From f18348e1b026c85c1f36604b9962100c2cfcfe20 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 26 Jul 2024 13:20:05 -0500 Subject: [PATCH 001/120] WIP --- .../trial_license_complete_tier/import_rules.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 526106a57df3c..025f8c9f41351 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1513,5 +1513,15 @@ export default ({ getService }: FtrProviderContext): void => { .expect(200); }); }); + + describe('supporting prebuilt rule customization', () => { + it('allows rules with "immutable: true"', async () => {}); + it('rejects rules without a rule_id', async () => {}); + + describe('calculation of the rule_source fields', () => { + it('calculates a version of 1 for custom rules'); + it('rejects a prebuilt rule with an unspecified version'); + }); + }); }); }; From ea65e92214975d1556ea7e1f146c138d35394fc7 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 21:51:38 -0500 Subject: [PATCH 002/120] Allow import endpoint to specify the immutable property This will be necessary for importing prebuilt rules without having to modify their contents. --- .../import_rules/rule_to_import.ts | 4 ++-- .../import_rules.ts | 24 ++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index 9dc1e218f6ceb..589dc316101d4 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -18,7 +18,7 @@ import { * Differences from this and the createRulesSchema are * - rule_id is required * - id is optional (but ignored in the import code - rule_id is exclusively used for imports) - * - immutable is optional but if it is any value other than false it will be rejected + * - immutable is optional (but ignored in the import code) * - created_at is optional (but ignored in the import code) * - updated_at is optional (but ignored in the import code) * - created_by is optional (but ignored in the import code) @@ -29,7 +29,7 @@ export type RuleToImportInput = z.input; export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( ResponseFields.partial().extend({ rule_id: RuleSignatureId, - immutable: z.literal(false).default(false), + immutable: z.boolean().optional(), /* Overriding `required_fields` from ResponseFields because in ResponseFields `required_fields` has the output type, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 025f8c9f41351..9647bd1d5f94d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1515,7 +1515,29 @@ export default ({ getService }: FtrProviderContext): void => { }); describe('supporting prebuilt rule customization', () => { - it('allows rules with "immutable: true"', async () => {}); + it('allows rules with "immutable: true"', async () => { + const rule = getCustomQueryRuleParams({ + rule_id: 'rule-immutable', + // TODO: should we allow "create" types to specify immutable, or do we now need a distinct "import" type to use here? + // @ts-expect-error + immutable: true, + }); + const ndjson = combineToNdJson(rule); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body).toEqual( + expect.objectContaining({ + success: true, + }) + ); + }); + it('rejects rules without a rule_id', async () => {}); describe('calculation of the rule_source fields', () => { From 179c2b46cb6ce10295c5563ac5cf33bab2d53fb6 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 22:05:38 -0500 Subject: [PATCH 003/120] Assert our requirement of rule_id This behavior was already there, we're just pinning it down. --- .../import_rules.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 9647bd1d5f94d..5a51780a6e54e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1538,7 +1538,25 @@ export default ({ getService }: FtrProviderContext): void => { ); }); - it('rejects rules without a rule_id', async () => {}); + it('rejects rules without a rule_id', async () => { + const rule = getCustomQueryRuleParams({}); + delete rule.rule_id; + const ndjson = combineToNdJson(rule); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body.errors).toHaveLength(1); + expect(body.errors[0]).toEqual( + expect.objectContaining({ + error: expect.objectContaining({ message: 'rule_id: Required', status_code: 400 }), + }) + ); + }); describe('calculation of the rule_source fields', () => { it('calculates a version of 1 for custom rules'); From 6360b0579753a43f3eee3466db396c7c00c609a1 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 22:11:34 -0500 Subject: [PATCH 004/120] Style: better assertion syntax Instead of nested uses of objectContaining with toEqual, we can instead use toMatchObject, which accomplishes the same. --- .../trial_license_complete_tier/import_rules.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 5a51780a6e54e..915f132a32dc1 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1531,11 +1531,9 @@ export default ({ getService }: FtrProviderContext): void => { .attach('file', Buffer.from(ndjson), 'rules.ndjson') .expect(200); - expect(body).toEqual( - expect.objectContaining({ - success: true, - }) - ); + expect(body).toMatchObject({ + success: true, + }); }); it('rejects rules without a rule_id', async () => { @@ -1551,11 +1549,9 @@ export default ({ getService }: FtrProviderContext): void => { .expect(200); expect(body.errors).toHaveLength(1); - expect(body.errors[0]).toEqual( - expect.objectContaining({ - error: expect.objectContaining({ message: 'rule_id: Required', status_code: 400 }), - }) - ); + expect(body.errors[0]).toMatchObject({ + error: { message: 'rule_id: Required', status_code: 400 }, + }); }); describe('calculation of the rule_source fields', () => { From 09001064ae0a7f5c6ff4cb4f9a4b987c58e2b2cc Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 22:15:54 -0500 Subject: [PATCH 005/120] I've decided that we don't need this type Since we only reference it in legacy validation. --- .../trial_license_complete_tier/import_rules.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 915f132a32dc1..66ab9e722d3a6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1518,8 +1518,7 @@ export default ({ getService }: FtrProviderContext): void => { it('allows rules with "immutable: true"', async () => { const rule = getCustomQueryRuleParams({ rule_id: 'rule-immutable', - // TODO: should we allow "create" types to specify immutable, or do we now need a distinct "import" type to use here? - // @ts-expect-error + // @ts-expect-error the API supports this param, but we only need it in {@link RuleToImport} immutable: true, }); const ndjson = combineToNdJson(rule); From 816fcfebc7425db39dcc1c77adceb72d78251b9f Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 22:28:11 -0500 Subject: [PATCH 006/120] Test our allowing of the rule_source param Like immutable, it's dropped, but we still support it being sent (so that users can send their prebuilt rules). --- .../import_rules.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 66ab9e722d3a6..998723bb1e809 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1535,6 +1535,28 @@ export default ({ getService }: FtrProviderContext): void => { }); }); + it('allows (but ignores) rules with a value for rule_source', async () => { + const rule = getCustomQueryRuleParams({ + rule_id: 'rule-source', + // @ts-expect-error the API supports this param, but we only need it in {@link RuleToImport} + rule_source: { + type: 'internal', + }, + }); + const ndjson = combineToNdJson(rule); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body).toMatchObject({ + success: true, + }); + }); + it('rejects rules without a rule_id', async () => { const rule = getCustomQueryRuleParams({}); delete rule.rule_id; From 70b53c1a67923d60b410099d4d8d8a75a4c89de0 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 22:29:50 -0500 Subject: [PATCH 007/120] Add some explanatory structure to our tests --- .../import_rules.ts | 96 ++++++++++--------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 998723bb1e809..c631414169893 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1515,63 +1515,65 @@ export default ({ getService }: FtrProviderContext): void => { }); describe('supporting prebuilt rule customization', () => { - it('allows rules with "immutable: true"', async () => { - const rule = getCustomQueryRuleParams({ - rule_id: 'rule-immutable', - // @ts-expect-error the API supports this param, but we only need it in {@link RuleToImport} - immutable: true, - }); - const ndjson = combineToNdJson(rule); + describe('compatibility with prebuilt rule fields', () => { + it('allows rules with "immutable: true"', async () => { + const rule = getCustomQueryRuleParams({ + rule_id: 'rule-immutable', + // @ts-expect-error the API supports this param, but we only need it in {@link RuleToImport} + immutable: true, + }); + const ndjson = combineToNdJson(rule); - const { body } = await supertest - .post(`${DETECTION_ENGINE_RULES_URL}/_import`) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '2023-10-31') - .attach('file', Buffer.from(ndjson), 'rules.ndjson') - .expect(200); + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); - expect(body).toMatchObject({ - success: true, + expect(body).toMatchObject({ + success: true, + }); }); - }); - it('allows (but ignores) rules with a value for rule_source', async () => { - const rule = getCustomQueryRuleParams({ - rule_id: 'rule-source', - // @ts-expect-error the API supports this param, but we only need it in {@link RuleToImport} - rule_source: { - type: 'internal', - }, - }); - const ndjson = combineToNdJson(rule); + it('allows (but ignores) rules with a value for rule_source', async () => { + const rule = getCustomQueryRuleParams({ + rule_id: 'rule-source', + // @ts-expect-error the API supports this param, but we only need it in {@link RuleToImport} + rule_source: { + type: 'internal', + }, + }); + const ndjson = combineToNdJson(rule); - const { body } = await supertest - .post(`${DETECTION_ENGINE_RULES_URL}/_import`) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '2023-10-31') - .attach('file', Buffer.from(ndjson), 'rules.ndjson') - .expect(200); + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); - expect(body).toMatchObject({ - success: true, + expect(body).toMatchObject({ + success: true, + }); }); - }); - it('rejects rules without a rule_id', async () => { - const rule = getCustomQueryRuleParams({}); - delete rule.rule_id; - const ndjson = combineToNdJson(rule); + it('rejects rules without a rule_id', async () => { + const rule = getCustomQueryRuleParams({}); + delete rule.rule_id; + const ndjson = combineToNdJson(rule); - const { body } = await supertest - .post(`${DETECTION_ENGINE_RULES_URL}/_import`) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '2023-10-31') - .attach('file', Buffer.from(ndjson), 'rules.ndjson') - .expect(200); + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); - expect(body.errors).toHaveLength(1); - expect(body.errors[0]).toMatchObject({ - error: { message: 'rule_id: Required', status_code: 400 }, + expect(body.errors).toHaveLength(1); + expect(body.errors[0]).toMatchObject({ + error: { message: 'rule_id: Required', status_code: 400 }, + }); }); }); From 7e531bb982cf213657c49dd355e3ad26305f1fae Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 22:30:05 -0500 Subject: [PATCH 008/120] Remove our optional field from the schema entirely This has the same effect: any value is allowed, but the field is ultimately ignored in the handling routes. --- .../rule_management/import_rules/rule_to_import.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index 589dc316101d4..2df09ceb3b803 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -29,7 +29,6 @@ export type RuleToImportInput = z.input; export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( ResponseFields.partial().extend({ rule_id: RuleSignatureId, - immutable: z.boolean().optional(), /* Overriding `required_fields` from ResponseFields because in ResponseFields `required_fields` has the output type, From 41b86b670c17d3623cfbcd777ca98049cd57c95e Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 22:30:53 -0500 Subject: [PATCH 009/120] Style: removing trailing whitespace --- .../rule_management/import_rules/rule_to_import.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index 2df09ceb3b803..0bf34aaff6d45 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -30,11 +30,11 @@ export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( ResponseFields.partial().extend({ rule_id: RuleSignatureId, /* - Overriding `required_fields` from ResponseFields because - in ResponseFields `required_fields` has the output type, + Overriding `required_fields` from ResponseFields because + in ResponseFields `required_fields` has the output type, but for importing rules, we need to use the input type. - Otherwise importing rules without the "ecs" property in - `required_fields` will fail. + Otherwise importing rules without the "ecs" property in + `required_fields` will fail. */ required_fields: z.array(RequiredFieldInput).optional(), }) From aaaf160e0185b9948dc7bb007c1306719cbb5f5c Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 22:36:34 -0500 Subject: [PATCH 010/120] Mark our immutable property as optional Our rule types may technically have this. --- .../lib/detection_engine/rule_schema/model/rule_schemas.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index 5d5065170deff..e3a9c4556653f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -125,7 +125,7 @@ export const BaseRuleParams = z.object({ from: RuleIntervalFrom, ruleId: RuleSignatureId, investigationFields: InvestigationFieldsCombined.optional(), - immutable: IsRuleImmutable, + immutable: IsRuleImmutable.optional(), ruleSource: RuleSourceCamelCased.optional(), license: RuleLicense.optional(), outputIndex: AlertsIndex, From a34fad5af433c38c7ac3b3e5c73f1445a1d889bd Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 22:53:07 -0500 Subject: [PATCH 011/120] Update rule schemas per RFC --- .../model/rule_schema/common_attributes.schema.yaml | 9 ++++++++- .../model/rule_schema/rule_schemas.schema.yaml | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml index 088153cca4885..1b3ebe9d89bb3 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml @@ -43,7 +43,12 @@ components: IsRuleImmutable: type: boolean deprecated: true - description: This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field. + description: '[DEPRECATION WARNING TODO] - This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field.' + + ExternalSourceUpdatedAt: + type: string + format: date-time + description: The date and time that the external/prebuilt rule was last updated in its source repository. IsExternalRuleCustomized: type: boolean @@ -70,6 +75,8 @@ components: - external is_customized: $ref: '#/components/schemas/IsExternalRuleCustomized' + source_updated_at: + $ref: '#/components/schemas/ExternalSourceUpdatedAt' required: - type - is_customized diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml index 4ade72c15fbb9..a6b82568df16d 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml @@ -203,6 +203,7 @@ components: - id - rule_id - immutable + - rule_source - updated_at - updated_by - created_at From 616b8d863ac9e1d481c91abbfe9ede64ed9e6744 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 23:00:04 -0500 Subject: [PATCH 012/120] Add an example datetime string to our openAPI schema As a user I always appreciate an actual example (for type, formatting, etc.) --- .../model/rule_schema/common_attributes.schema.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml index 1b3ebe9d89bb3..aa56be14120ec 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml @@ -48,6 +48,7 @@ components: ExternalSourceUpdatedAt: type: string format: date-time + example: '2021-08-31T00:00:00Z' description: The date and time that the external/prebuilt rule was last updated in its source repository. IsExternalRuleCustomized: From 7a16826fd88d800339ebde2f29f6163058b2060f Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 5 Aug 2024 23:00:41 -0500 Subject: [PATCH 013/120] Add our new, optional field to prebuilt asset schema This will be added to the detection-rules repo as they modify rules moving forward. --- .../prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index c41d24660462e..980b1c704695b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -72,6 +72,7 @@ function zodMaskFor() { * Big differences between this schema and RuleCreateProps: * - rule_id is a required field * - version is a required field + * - source_updated_at is a new, optional field in support of prebuilt rule customization * - some fields are omitted because they are not present in https://github.com/elastic/detection-rules */ export type PrebuiltRuleAsset = z.infer; @@ -81,5 +82,6 @@ export const PrebuiltRuleAsset = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PR z.object({ rule_id: RuleSignatureId, version: RuleVersion, + source_updated_at: z.string().datetime().optional(), }) ); From cbeef5ca9413d18aac7fc8b54b3962ba45eecbd9 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 8 Aug 2024 16:27:36 -0500 Subject: [PATCH 014/120] Reject prebuilt rule import while feature flag is disabled In order to prevent users from importing prebuilt rules before the feature is fully enabled, we need to add additional validation logic to account for our looser schema validation (which allows any value for `immutable`). --- .../api/rules/import_rules/route.ts | 1 + .../logic/import/import_rules_utils.test.ts | 53 +++++++++++++++++++ .../logic/import/import_rules_utils.ts | 21 ++++++++ .../import_rules.ts | 48 ++++++++++++++++- 4 files changed, 121 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index bf62101f2f5a1..8966695308ff1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -158,6 +158,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C detectionRulesClient, existingLists: foundReferencedExceptionLists, allowMissingConnectorSecrets: !!actionConnectors.length, + allowPrebuiltRules: config.experimentalFeatures.prebuiltRulesCustomizationEnabled, }); const errorsResp = importRuleResponse.filter((resp) => isBulkError(resp)) as BulkError[]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts index 6537dfed3a169..2a694af83003c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts @@ -98,4 +98,57 @@ describe('importRules', () => { expect(result).toEqual([{ rule_id: ruleToImport.rule_id, status_code: 200 }]); }); + + describe('compatibility with prebuilt rules', () => { + let prebuiltRuleToImport: ReturnType; + + beforeEach(() => { + prebuiltRuleToImport = { + ...getImportRulesSchemaMock(), + immutable: true, + }; + }); + + it('rejects a prebuilt rule when allowPrebuiltRules is false', async () => { + const ruleChunk = [prebuiltRuleToImport]; + const result = await importRules({ + ruleChunks: [ruleChunk], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + existingLists: {}, + allowPrebuiltRules: false, + }); + + expect(result).toEqual([ + { + error: { + message: + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, remove its "immutable" property try again.', + status_code: 400, + }, + rule_id: prebuiltRuleToImport.rule_id, + }, + ]); + }); + + it('imports a prebuilt rule when allowPrebuiltRules is true', async () => { + clients.detectionRulesClient.importRule.mockResolvedValue({ + ...getRulesSchemaMock(), + rule_id: prebuiltRuleToImport.rule_id, + }); + + const ruleChunk = [prebuiltRuleToImport]; + const result = await importRules({ + ruleChunks: [ruleChunk], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + existingLists: {}, + allowPrebuiltRules: true, + }); + + expect(result).toEqual([{ rule_id: prebuiltRuleToImport.rule_id, status_code: 200 }]); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 5fc57a7a91e45..1c45aeb1d3f76 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { i18n } from '@kbn/i18n'; import type { SavedObject } from '@kbn/core/server'; import type { ImportExceptionsListSchema, @@ -46,6 +47,7 @@ export const importRules = async ({ detectionRulesClient, existingLists, allowMissingConnectorSecrets, + allowPrebuiltRules, }: { ruleChunks: PromiseFromStreams[][]; rulesResponseAcc: ImportRuleResponse[]; @@ -53,6 +55,7 @@ export const importRules = async ({ detectionRulesClient: IDetectionRulesClient; existingLists: Record; allowMissingConnectorSecrets?: boolean; + allowPrebuiltRules?: boolean; }) => { let importRuleResponse: ImportRuleResponse[] = [...rulesResponseAcc]; @@ -80,6 +83,24 @@ export const importRules = async ({ return null; } + if (!allowPrebuiltRules && parsedRule.immutable) { + resolve( + createBulkErrorObject({ + statusCode: 400, + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.importPrebuiltRulesUnsupported', + { + defaultMessage: + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, remove its "immutable" property try again.', + } + ), + ruleId: parsedRule.rule_id, + }) + ); + + return null; + } + try { const [exceptionErrors, exceptions] = checkRuleExceptionReferences({ rule: parsedRule, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index c631414169893..260e1e1ca7944 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1516,7 +1516,7 @@ export default ({ getService }: FtrProviderContext): void => { describe('supporting prebuilt rule customization', () => { describe('compatibility with prebuilt rule fields', () => { - it('allows rules with "immutable: true"', async () => { + it('rejects rules with "immutable: true" when the feature flag is disabled', async () => { const rule = getCustomQueryRuleParams({ rule_id: 'rule-immutable', // @ts-expect-error the API supports this param, but we only need it in {@link RuleToImport} @@ -1532,7 +1532,51 @@ export default ({ getService }: FtrProviderContext): void => { .expect(200); expect(body).toMatchObject({ - success: true, + success: false, + errors: [ + { + rule_id: 'rule-immutable', + error: { + status_code: 400, + message: + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, remove its "immutable" property try again.', + }, + }, + ], + }); + }); + + it('imports custom rules alongside prebuilt rules when feature flag is disabled', async () => { + const ndjson = combineToNdJson( + getCustomQueryRuleParams({ + rule_id: 'rule-immutable', + // @ts-expect-error the API supports the 'immutable' param, but we only need it in {@link RuleToImport} + immutable: true, + }), + // @ts-expect-error the API supports the 'immutable' param, but we only need it in {@link RuleToImport} + getCustomQueryRuleParams({ rule_id: 'custom-rule', immutable: false }) + ); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body).toMatchObject({ + success: false, + success_count: 1, + errors: [ + { + rule_id: 'rule-immutable', + error: { + status_code: 400, + message: + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, remove its "immutable" property try again.', + }, + }, + ], }); }); From 8dd304f47195994ee934eac3f86a19483e9e13c7 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 8 Aug 2024 16:45:45 -0500 Subject: [PATCH 015/120] More comprehensive test That actually asserts that our specified rule_source was ignored. --- .../trial_license_complete_tier/import_rules.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 260e1e1ca7944..a1f86fcce73ff 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1582,10 +1582,10 @@ export default ({ getService }: FtrProviderContext): void => { it('allows (but ignores) rules with a value for rule_source', async () => { const rule = getCustomQueryRuleParams({ - rule_id: 'rule-source', + rule_id: 'with-rule-source', // @ts-expect-error the API supports this param, but we only need it in {@link RuleToImport} rule_source: { - type: 'internal', + type: 'ignored', }, }); const ndjson = combineToNdJson(rule); @@ -1599,7 +1599,12 @@ export default ({ getService }: FtrProviderContext): void => { expect(body).toMatchObject({ success: true, + success_count: 1, }); + + const importedRule = await fetchRule(supertest, { ruleId: 'with-rule-source' }); + + expect(importedRule.rule_source).toMatchObject({ type: 'internal' }); }); it('rejects rules without a rule_id', async () => { From 28fc492f28a54138b0f333930e5835d437582e34 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 8 Aug 2024 17:03:19 -0500 Subject: [PATCH 016/120] WIP: ensuring prebuilt rules package is installed when importing rules The logic here can/should be optimized, but this is mostly just leveraging existing helpers to do this work. --- .../rule_management/api/rules/import_rules/route.ts | 12 +++++++++++- .../trial_license_complete_tier/import_rules.ts | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 8966695308ff1..0af1830d01faf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -21,6 +21,8 @@ import type { ConfigType } from '../../../../../../config'; import type { HapiReadableStream, SecuritySolutionPluginRouter } from '../../../../../../types'; import type { BulkError, ImportRuleResponse } from '../../../../routes/utils'; import { buildSiemResponse, isBulkError, isImportRegular } from '../../../../routes/utils'; +import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; +import { ensureLatestRulesPackageInstalled } from '../../../../prebuilt_rules/logic/ensure_latest_rules_package_installed'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; import { createRulesAndExceptionsStreamFromNdJson } from '../../../logic/import/create_rules_stream_from_ndjson'; import { getReferencedExceptionLists } from '../../../logic/import/gather_referenced_exceptions'; @@ -74,6 +76,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C 'licensing', ]); + const { prebuiltRulesCustomizationEnabled } = config.experimentalFeatures; const detectionRulesClient = ctx.securitySolution.getDetectionRulesClient(); const actionsClient = ctx.actions.getActionsClient(); const actionSOClient = ctx.core.savedObjects.getClient({ @@ -101,6 +104,13 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C RuleExceptionsPromiseFromStreams[] >([request.body.file as HapiReadableStream, ...readAllStream]); + // TODO: optimize this so we only check/install if prebuilt rules are being imported? + if (prebuiltRulesCustomizationEnabled) { + const ruleAssetsClient = createPrebuiltRuleAssetsClient(savedObjectsClient); + await ensureLatestRulesPackageInstalled(ruleAssetsClient, config, ctx.securitySolution); + // TODO: calculate rule source, remove immutable field + } + // import exceptions, includes validation const { errors: exceptionsErrors, @@ -158,7 +168,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C detectionRulesClient, existingLists: foundReferencedExceptionLists, allowMissingConnectorSecrets: !!actionConnectors.length, - allowPrebuiltRules: config.experimentalFeatures.prebuiltRulesCustomizationEnabled, + allowPrebuiltRules: prebuiltRulesCustomizationEnabled, }); const errorsResp = importRuleResponse.filter((resp) => isBulkError(resp)) as BulkError[]; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index a1f86fcce73ff..a160c5e70869b 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1624,6 +1624,8 @@ export default ({ getService }: FtrProviderContext): void => { error: { message: 'rule_id: Required', status_code: 400 }, }); }); + + it('installs prebuilt rules package if it is not installed'); }); describe('calculation of the rule_source fields', () => { From edda029c923a303251e7347103afc78d921d905d Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 8 Aug 2024 22:56:25 +0000 Subject: [PATCH 017/120] [CI] Auto-commit changed files from 'yarn openapi:generate' --- .../model/rule_schema/common_attributes.gen.ts | 9 ++++++++- .../model/rule_schema/rule_schemas.gen.ts | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts index 126b11bfa6cdb..bf5a9a30d25c1 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts @@ -51,12 +51,18 @@ export type KqlQueryLanguageEnum = typeof KqlQueryLanguage.enum; export const KqlQueryLanguageEnum = KqlQueryLanguage.enum; /** - * This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field. + * [DEPRECATION WARNING TODO] - This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field. * @deprecated */ export type IsRuleImmutable = z.infer; export const IsRuleImmutable = z.boolean(); +/** + * The date and time that the external/prebuilt rule was last updated in its source repository. + */ +export type ExternalSourceUpdatedAt = z.infer; +export const ExternalSourceUpdatedAt = z.string().datetime(); + /** * Determines whether an external/prebuilt rule has been customized by the user (i.e. any of its fields have been modified and diverged from the base value). */ @@ -78,6 +84,7 @@ export type ExternalRuleSource = z.infer; export const ExternalRuleSource = z.object({ type: z.literal('external'), is_customized: IsExternalRuleCustomized, + source_updated_at: ExternalSourceUpdatedAt.optional(), }); /** diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index 83bf6778ec3e3..8193ae82fa8ff 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -160,7 +160,7 @@ export const ResponseFields = z.object({ id: RuleObjectId, rule_id: RuleSignatureId, immutable: IsRuleImmutable, - rule_source: RuleSource.optional(), + rule_source: RuleSource, updated_at: z.string().datetime(), updated_by: z.string(), created_at: z.string().datetime(), From b5e36aae5dab8345022c70578c465707f4a77d97 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 8 Aug 2024 20:45:58 -0500 Subject: [PATCH 018/120] Revert making a base rule param (immutable) optional It would be a breaking change to make this param optional. Since we've discussed keeping it around but deprecating it, we don't really need to change this type, and need to instead ensure that we create this field if left unspecified. --- .../lib/detection_engine/rule_schema/model/rule_schemas.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index e3a9c4556653f..5d5065170deff 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -125,7 +125,7 @@ export const BaseRuleParams = z.object({ from: RuleIntervalFrom, ruleId: RuleSignatureId, investigationFields: InvestigationFieldsCombined.optional(), - immutable: IsRuleImmutable.optional(), + immutable: IsRuleImmutable, ruleSource: RuleSourceCamelCased.optional(), license: RuleLicense.optional(), outputIndex: AlertsIndex, From c7fc008bcc8afbb4f4a13525e86a4ee9ed4a97ba Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 8 Aug 2024 20:50:18 -0500 Subject: [PATCH 019/120] Default immutable to false in our importing path This change to the RuleToImport schema affects all rules being imported, as they're validated by this schema. This ensures we always generate a value for the field, which is needed for backwards compatibility. Associated unit tests were similarly updated. --- .../import_rules/rule_to_import.test.ts | 12 ++++-------- .../rule_management/import_rules/rule_to_import.ts | 1 + 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts index 359dad3235475..8e7415b7c2729 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts @@ -560,7 +560,7 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"immutable: Invalid literal value, expected false"` + `"immutable: Expected boolean, received number"` ); }); @@ -574,18 +574,14 @@ describe('RuleToImport', () => { expectParseSuccess(result); }); - test('You cannot set the immutable to be true', () => { + test('You can optionally set the immutable to be true', () => { const payload = getImportRulesSchemaMock({ - // @ts-expect-error assign unsupported value immutable: true, }); const result = RuleToImport.safeParse(payload); - expectParseError(result); - expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"immutable: Invalid literal value, expected false"` - ); + expectParseSuccess(result); }); test('You cannot set the immutable to be a number', () => { @@ -598,7 +594,7 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"immutable: Invalid literal value, expected false"` + `"immutable: Expected boolean, received number"` ); }); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index 0bf34aaff6d45..f7ab252171667 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -29,6 +29,7 @@ export type RuleToImportInput = z.input; export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( ResponseFields.partial().extend({ rule_id: RuleSignatureId, + immutable: z.boolean().optional().default(false), /* Overriding `required_fields` from ResponseFields because in ResponseFields `required_fields` has the output type, From 79cbf0e0347827e010aa96f6c85e17d2721b71c7 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 9 Aug 2024 02:43:22 +0000 Subject: [PATCH 020/120] [CI] Auto-commit changed files from 'yarn openapi:bundle' --- ...n_detections_api_2023_10_31.bundled.schema.yaml | 14 ++++++++++++-- ...n_detections_api_2023_10_31.bundled.schema.yaml | 14 ++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml index c9bb81ede9be3..155e36d436f2c 100644 --- a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -2967,6 +2967,8 @@ components: properties: is_customized: $ref: '#/components/schemas/IsExternalRuleCustomized' + source_updated_at: + $ref: '#/components/schemas/ExternalSourceUpdatedAt' type: enum: - external @@ -2974,6 +2976,13 @@ components: required: - type - is_customized + ExternalSourceUpdatedAt: + description: >- + The date and time that the external/prebuilt rule was last updated in + its source repository. + example: '2021-08-31T00:00:00Z' + format: date-time + type: string FindRulesSortField: enum: - created_at @@ -3085,8 +3094,8 @@ components: IsRuleImmutable: deprecated: true description: >- - This field determines whether the rule is a prebuilt Elastic rule. It - will be replaced with the `rule_source` field. + [DEPRECATION WARNING TODO] - This field determines whether the rule is a + prebuilt Elastic rule. It will be replaced with the `rule_source` field. type: boolean ItemsPerSearch: minimum: 1 @@ -4868,6 +4877,7 @@ components: - id - rule_id - immutable + - rule_source - updated_at - updated_by - created_at diff --git a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml index 6a8f72d45d46c..bfffb2a081a2e 100644 --- a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -2242,6 +2242,8 @@ components: properties: is_customized: $ref: '#/components/schemas/IsExternalRuleCustomized' + source_updated_at: + $ref: '#/components/schemas/ExternalSourceUpdatedAt' type: enum: - external @@ -2249,6 +2251,13 @@ components: required: - type - is_customized + ExternalSourceUpdatedAt: + description: >- + The date and time that the external/prebuilt rule was last updated in + its source repository. + example: '2021-08-31T00:00:00Z' + format: date-time + type: string FindRulesSortField: enum: - created_at @@ -2337,8 +2346,8 @@ components: IsRuleImmutable: deprecated: true description: >- - This field determines whether the rule is a prebuilt Elastic rule. It - will be replaced with the `rule_source` field. + [DEPRECATION WARNING TODO] - This field determines whether the rule is a + prebuilt Elastic rule. It will be replaced with the `rule_source` field. type: boolean ItemsPerSearch: minimum: 1 @@ -4022,6 +4031,7 @@ components: - id - rule_id - immutable + - rule_source - updated_at - updated_by - created_at From 1c5834b71d4c79f0234168f2f0ef8a4df092392e Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 8 Aug 2024 21:18:08 -0500 Subject: [PATCH 021/120] Update mocks and tests to reflect required param rule_source rule_source is now a required param on rule output, so a lot of things are gonna need to be fixed. --- .../model/rule_schema/rule_response_schema.mock.ts | 1 + .../model/rule_schema/rule_response_schema.test.ts | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.mock.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.mock.ts index c605436576995..59b09533a6e14 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.mock.ts @@ -47,6 +47,7 @@ const getResponseBaseParams = (anchorDate: string = ANCHOR_DATE): SharedResponse risk_score: 55, risk_score_mapping: [], rule_id: 'query-rule-id', + rule_source: { type: 'internal' }, interval: '5m', exceptions_list: getListArrayMock(), related_integrations: [], diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts index 30fe84514e05a..9546ab3a59b09 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts @@ -266,12 +266,13 @@ describe('rule_source', () => { expect(result.data).toEqual(payload); }); - test('it should validate a rule with "rule_source" set to undefined', () => { + test('it should not validate a rule with "rule_source" set to undefined', () => { const payload = getRulesSchemaMock(); - payload.rule_source = undefined; + // @ts-expect-error + delete payload.rule_source; const result = RuleResponse.safeParse(payload); - expectParseSuccess(result); - expect(result.data).toEqual(payload); + expectParseError(result); + expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"rule_source: Required"`); }); }); From 17f68a39d28f217b59df5f9c867b94702af7dcb0 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 9 Aug 2024 12:51:24 -0500 Subject: [PATCH 022/120] Apply Dmitrii's solution to type incompatibilities This conditionally calls our conversion utilities so that we always act with data, and have more accurate return types on those helpers. I think we can make this even better, but let's first see if this addresses the type issues. --- .../common_params_camel_to_snake.ts | 5 +++- .../convert_rule_response_to_alerting_rule.ts | 28 ++++++++++++++----- .../internal_rule_to_api_response.ts | 15 +++++----- .../converters/normalize_rule_params.ts | 17 ++++++++--- .../type_specific_camel_to_snake.ts | 28 ++++++++++++++----- .../server/utils/object_case_converters.ts | 14 ++-------- 6 files changed, 68 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts index 6f98230043e74..355f49aba3ee7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts @@ -9,6 +9,9 @@ import { convertObjectKeysToSnakeCase } from '../../../../../../utils/object_cas import type { BaseRuleParams } from '../../../../rule_schema'; import { migrateLegacyInvestigationFields } from '../../../utils/utils'; +/** + * @deprecated Use convertObjectKeysToSnakeCase instead + */ export const commonParamsCamelToSnake = (params: BaseRuleParams) => { return { description: params.description, @@ -39,7 +42,7 @@ export const commonParamsCamelToSnake = (params: BaseRuleParams) => { version: params.version, exceptions_list: params.exceptionsList, immutable: params.immutable, - rule_source: convertObjectKeysToSnakeCase(params.ruleSource), + rule_source: params.ruleSource ? convertObjectKeysToSnakeCase(params.ruleSource) : undefined, related_integrations: params.relatedIntegrations ?? [], required_fields: params.requiredFields ?? [], setup: params.setup ?? '', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts index 52a00f1d6f42d..98e5676c031c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts @@ -117,7 +117,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific timestampField: params.timestamp_field, eventCategoryOverride: params.event_category_override, tiebreakerField: params.tiebreaker_field, - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'esql': { @@ -125,7 +127,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific type: params.type, language: params.language, query: params.query, - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'threat_match': { @@ -145,7 +149,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific threatIndicatorPath: params.threat_indicator_path, concurrentSearches: params.concurrent_searches, itemsPerSearch: params.items_per_search, - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'query': { @@ -160,7 +166,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific responseActions: params.response_actions?.map((rule) => transformRuleToAlertResponseAction(rule) ), - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'saved_query': { @@ -175,7 +183,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific responseActions: params.response_actions?.map((rule) => transformRuleToAlertResponseAction(rule) ), - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'threshold': { @@ -198,7 +208,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific type: params.type, anomalyThreshold: params.anomaly_threshold, machineLearningJobId: normalizeMachineLearningJobIds(params.machine_learning_job_id), - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } case 'new_terms': { @@ -211,7 +223,9 @@ const typeSpecificSnakeToCamel = (params: TypeSpecificCreateProps): TypeSpecific filters: params.filters, language: params.language ?? 'kuery', dataViewId: params.data_view_id, - alertSuppression: convertObjectKeysToCamelCase(params.alert_suppression), + alertSuppression: params.alert_suppression + ? convertObjectKeysToCamelCase(params.alert_suppression) + : undefined, }; } default: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts index 336cd8fae0405..d364066be56b7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts @@ -6,25 +6,24 @@ */ import type { ResolvedSanitizedRule, SanitizedRule } from '@kbn/alerting-plugin/common'; -import type { RequiredOptional } from '@kbn/zod-helpers'; import type { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; import { transformAlertToRuleAction, transformAlertToRuleSystemAction, } from '../../../../../../../common/detection_engine/transform_actions'; import { createRuleExecutionSummary } from '../../../../rule_monitoring'; -import type { RuleParams } from '../../../../rule_schema'; +import { type RuleParams } from '../../../../rule_schema'; import { transformFromAlertThrottle, transformToActionFrequency, } from '../../../normalization/rule_actions'; import { typeSpecificCamelToSnake } from './type_specific_camel_to_snake'; -import { commonParamsCamelToSnake } from './common_params_camel_to_snake'; import { normalizeRuleParams } from './normalize_rule_params'; +import { convertObjectKeysToSnakeCase } from '../../../../../../utils/object_case_converters'; export const internalRuleToAPIResponse = ( rule: SanitizedRule | ResolvedSanitizedRule -): RequiredOptional => { +): RuleResponse => { const executionSummary = createRuleExecutionSummary(rule); const isResolvedRule = (obj: unknown): obj is ResolvedSanitizedRule => { @@ -40,6 +39,8 @@ export const internalRuleToAPIResponse = ( return transformedAction; }); const normalizedRuleParams = normalizeRuleParams(rule.params); + const commonRuleParams = convertObjectKeysToSnakeCase(normalizedRuleParams); + const typeSpecificRuleParams = typeSpecificCamelToSnake(rule.params); return { // saved object properties @@ -57,10 +58,8 @@ export const internalRuleToAPIResponse = ( interval: rule.schedule.interval, enabled: rule.enabled, revision: rule.revision, - // Security solution shared rule params - ...commonParamsCamelToSnake(normalizedRuleParams), - // Type specific security solution rule params - ...typeSpecificCamelToSnake(rule.params), + ...commonRuleParams, + ...typeSpecificRuleParams, // Actions throttle: undefined, actions: [...actions, ...(systemActions ?? [])], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts index eddd8b0434ba0..7281de4dc4586 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts @@ -5,6 +5,7 @@ * 2.0. */ import type { BaseRuleParams, RuleSourceCamelCased } from '../../../../rule_schema'; +import { migrateLegacyInvestigationFields } from '../../../utils/utils'; interface NormalizeRuleSourceParams { immutable: BaseRuleParams['immutable']; @@ -37,12 +38,20 @@ export const normalizeRuleSource = ({ }; export const normalizeRuleParams = (params: BaseRuleParams) => { + const investigationFields = migrateLegacyInvestigationFields(params.investigationFields); + const ruleSource = normalizeRuleSource({ + immutable: params.immutable, + ruleSource: params.ruleSource, + }); + return { ...params, + // These fields are types as optional in the data model, but they are required in our domain + setup: params.setup ?? '', + relatedIntegrations: params.relatedIntegrations ?? [], + requiredFields: params.requiredFields ?? [], // Fields to normalize - ruleSource: normalizeRuleSource({ - immutable: params.immutable, - ruleSource: params.ruleSource, - }), + investigationFields, + ruleSource, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/type_specific_camel_to_snake.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/type_specific_camel_to_snake.ts index 0808d1921e9bf..da7f4d056585e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/type_specific_camel_to_snake.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/type_specific_camel_to_snake.ts @@ -27,7 +27,9 @@ export const typeSpecificCamelToSnake = ( timestamp_field: params.timestampField, event_category_override: params.eventCategoryOverride, tiebreaker_field: params.tiebreakerField, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'esql': { @@ -35,7 +37,9 @@ export const typeSpecificCamelToSnake = ( type: params.type, language: params.language, query: params.query, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'threat_match': { @@ -55,7 +59,9 @@ export const typeSpecificCamelToSnake = ( threat_indicator_path: params.threatIndicatorPath, concurrent_searches: params.concurrentSearches, items_per_search: params.itemsPerSearch, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'query': { @@ -68,7 +74,9 @@ export const typeSpecificCamelToSnake = ( filters: params.filters, saved_id: params.savedId, response_actions: params.responseActions?.map(transformAlertToRuleResponseAction), - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'saved_query': { @@ -81,7 +89,9 @@ export const typeSpecificCamelToSnake = ( saved_id: params.savedId, data_view_id: params.dataViewId, response_actions: params.responseActions?.map(transformAlertToRuleResponseAction), - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'threshold': { @@ -104,7 +114,9 @@ export const typeSpecificCamelToSnake = ( type: params.type, anomaly_threshold: params.anomalyThreshold, machine_learning_job_id: params.machineLearningJobId, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } case 'new_terms': { @@ -117,7 +129,9 @@ export const typeSpecificCamelToSnake = ( filters: params.filters, language: params.language, data_view_id: params.dataViewId, - alert_suppression: convertObjectKeysToSnakeCase(params.alertSuppression), + alert_suppression: params.alertSuppression + ? convertObjectKeysToSnakeCase(params.alertSuppression) + : undefined, }; } default: { diff --git a/x-pack/plugins/security_solution/server/utils/object_case_converters.ts b/x-pack/plugins/security_solution/server/utils/object_case_converters.ts index ebe9158fe9c99..73aed4e8af0f9 100644 --- a/x-pack/plugins/security_solution/server/utils/object_case_converters.ts +++ b/x-pack/plugins/security_solution/server/utils/object_case_converters.ts @@ -8,20 +8,10 @@ import camelcaseKeys from 'camelcase-keys'; import snakecaseKeys from 'snakecase-keys'; import type { CamelCasedPropertiesDeep, SnakeCasedPropertiesDeep } from 'type-fest'; -export const convertObjectKeysToCamelCase = >( - obj: T | undefined -) => { - if (obj == null) { - return obj; - } +export const convertObjectKeysToCamelCase = >(obj: T) => { return camelcaseKeys(obj, { deep: true }) as unknown as CamelCasedPropertiesDeep; }; -export const convertObjectKeysToSnakeCase = >( - obj: T | undefined -) => { - if (obj == null) { - return obj; - } +export const convertObjectKeysToSnakeCase = >(obj: T) => { return snakecaseKeys(obj, { deep: true }) as SnakeCasedPropertiesDeep; }; From d0e7f3eb790a15e751d2c52928fc504147f41b84 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 13 Aug 2024 14:30:30 -0500 Subject: [PATCH 023/120] Add test for added method functionality --- .../converters/normalize_rule_params.test.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts index b8b5db137583b..8398bab4253f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { normalizeRuleSource } from './normalize_rule_params'; +import { normalizeRuleSource, normalizeRuleParams } from './normalize_rule_params'; import type { BaseRuleParams } from '../../../../rule_schema'; describe('normalizeRuleSource', () => { @@ -53,3 +53,14 @@ describe('normalizeRuleSource', () => { }); }); }); + +describe('normalizeRuleParams', () => { + it('migrates legacy investigation fields', () => { + const params = { + investigationFields: ['field_1', 'field_2'], + } as BaseRuleParams; + const result = normalizeRuleParams(params); + + expect(result.investigationFields).toMatchObject({ field_names: ['field_1', 'field_2'] }); + }); +}); From 88d912841eafa52c3daac57824557f35d8abf715 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 13 Aug 2024 16:18:38 -0500 Subject: [PATCH 024/120] Remove unneeded cast --- .../security_solution/server/utils/object_case_converters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/utils/object_case_converters.ts b/x-pack/plugins/security_solution/server/utils/object_case_converters.ts index 73aed4e8af0f9..6a32491767bb5 100644 --- a/x-pack/plugins/security_solution/server/utils/object_case_converters.ts +++ b/x-pack/plugins/security_solution/server/utils/object_case_converters.ts @@ -9,7 +9,7 @@ import snakecaseKeys from 'snakecase-keys'; import type { CamelCasedPropertiesDeep, SnakeCasedPropertiesDeep } from 'type-fest'; export const convertObjectKeysToCamelCase = >(obj: T) => { - return camelcaseKeys(obj, { deep: true }) as unknown as CamelCasedPropertiesDeep; + return camelcaseKeys(obj, { deep: true }) as CamelCasedPropertiesDeep; }; export const convertObjectKeysToSnakeCase = >(obj: T) => { From 52daf293ca2f6dacd52e21e952e58c985a4f7905 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 13 Aug 2024 17:12:29 -0500 Subject: [PATCH 025/120] Test behavior when prebuilt rules installation fails --- .../api/rules/import_rules/route.test.ts | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts index 123b39a588c59..c9bc11384afa1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts @@ -15,7 +15,7 @@ import { import { getRulesSchemaMock } from '../../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; import type { requestMock } from '../../../../routes/__mocks__'; -import { createMockConfig, requestContextMock, serverMock } from '../../../../routes/__mocks__'; +import { configMock, requestContextMock, serverMock } from '../../../../routes/__mocks__'; import { buildHapiStream } from '../../../../routes/__mocks__/utils'; import { getImportRulesRequest, @@ -30,11 +30,18 @@ import * as createRulesAndExceptionsStreamFromNdJson from '../../../logic/import import { getQueryRuleParams } from '../../../../rule_schema/mocks'; import { importRulesRoute } from './route'; import { HttpAuthzError } from '../../../../../machine_learning/validation'; +import { createPrebuiltRuleAssetsClient as createPrebuiltRuleAssetsClientMock } from '../../../../prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client'; jest.mock('../../../../../machine_learning/authz'); +let mockPrebuiltRuleAssetsClient: ReturnType; + +jest.mock('../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client', () => ({ + createPrebuiltRuleAssetsClient: () => mockPrebuiltRuleAssetsClient, +})); + describe('Import rules route', () => { - let config: ReturnType; + let config: ReturnType; let server: ReturnType; let request: ReturnType; let { clients, context } = requestContextMock.createTools(); @@ -42,7 +49,7 @@ describe('Import rules route', () => { beforeEach(() => { server = serverMock.create(); ({ clients, context } = requestContextMock.createTools()); - config = createMockConfig(); + config = configMock.createDefault(); const hapiStream = buildHapiStream(ruleIdsToNdJsonString(['rule-1'])); request = getImportRulesRequest(hapiStream); @@ -54,6 +61,7 @@ describe('Import rules route', () => { context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValue( elasticsearchClientMock.createSuccessTransportRequestPromise(getBasicEmptySearchResponse()) ); + mockPrebuiltRuleAssetsClient = createPrebuiltRuleAssetsClientMock(); importRulesRoute(server.router, config); }); @@ -133,6 +141,27 @@ describe('Import rules route', () => { expect(response.status).toEqual(400); expect(response.body).toEqual({ message: 'Invalid file extension .html', status_code: 400 }); }); + + describe('with prebuilt rules customization enabled', () => { + beforeEach(() => { + server = serverMock.create(); // old server already registered this route + config = configMock.withExperimentalFeature(config, 'prebuiltRulesCustomizationEnabled'); + + importRulesRoute(server.router, config); + }); + + test('returns 500 if prebuilt rule installation fails', async () => { + mockPrebuiltRuleAssetsClient.fetchLatestAssets.mockRejectedValue(new Error('test error')); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + + expect(response.status).toEqual(500); + expect(response.body).toMatchObject({ + message: 'test error', + status_code: 500, + }); + }); + }); }); describe('single rule import', () => { From e5ccf7b4f23c7ad21f39054d546f33d2cf829437 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 13 Aug 2024 23:04:57 -0500 Subject: [PATCH 026/120] Adding in prebuilt rule assets to our import logic * Extracts preparation steps to a new helper * Adds assets param to import helper * Adds new case where rule_id is specified with no version For the last point, I've added an API test, but it won't work until I enable the FF for that test. I'm also not sure that anything's in the right place, but I'm currently just trying to get it functioning as specified. --- .../prepare_prebuilt_rules_for_import.ts | 54 +++++++++++++++++++ .../api/rules/import_rules/route.ts | 18 +++---- .../logic/import/import_rules_utils.ts | 22 ++++++++ .../import_rules.ts | 22 ++++++-- 4 files changed, 104 insertions(+), 12 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prepare_prebuilt_rules_for_import.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prepare_prebuilt_rules_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prepare_prebuilt_rules_for_import.ts new file mode 100644 index 0000000000000..ec739dfe9bac9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prepare_prebuilt_rules_for_import.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from '@kbn/core/server'; +import type { SecuritySolutionApiRequestHandlerContext } from '../../../../types'; +import type { RuleToImport } from '../../../../../common/api/detection_engine'; +import type { ConfigType } from '../../../../config'; +import type { PrebuiltRuleAsset } from '../model/rule_assets/prebuilt_rule_asset'; +import { createPrebuiltRuleAssetsClient } from './rule_assets/prebuilt_rule_assets_client'; +import { ensureLatestRulesPackageInstalled } from './ensure_latest_rules_package_installed'; + +/** + * + * Prepares the system to import prebuilt rules by ensuring the latest rules + * package is installed and fetching the corresponding prebuilt rule assets from the package. + * @param rules - The rules to be imported + * + * @returns The prebuilt rule assets corresponding to the specified prebuilt + * rules, which are used to determine how to import those rules (create vs. update, etc.). + */ +export const preparePrebuiltRuleAssetsForImport = async ({ + savedObjectsClient, + config, + context, + rules, +}: { + config: ConfigType; + context: SecuritySolutionApiRequestHandlerContext; + savedObjectsClient: SavedObjectsClientContract; + rules: Array; +}): Promise => { + if (!config.experimentalFeatures.prebuiltRulesCustomizationEnabled) { + return []; + } + + const ruleAssetsClient = createPrebuiltRuleAssetsClient(savedObjectsClient); + await ensureLatestRulesPackageInstalled(ruleAssetsClient, config, context); + + const prebuiltRulesToImport = rules.flatMap((rule) => { + if (rule instanceof Error || rule.version == null) { + return []; + } + return { + rule_id: rule.rule_id, + version: rule.version, + }; + }); + + return ruleAssetsClient.fetchAssetsByVersion(prebuiltRulesToImport); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 0af1830d01faf..e17742ab78f85 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -21,8 +21,7 @@ import type { ConfigType } from '../../../../../../config'; import type { HapiReadableStream, SecuritySolutionPluginRouter } from '../../../../../../types'; import type { BulkError, ImportRuleResponse } from '../../../../routes/utils'; import { buildSiemResponse, isBulkError, isImportRegular } from '../../../../routes/utils'; -import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; -import { ensureLatestRulesPackageInstalled } from '../../../../prebuilt_rules/logic/ensure_latest_rules_package_installed'; +import { preparePrebuiltRuleAssetsForImport } from '../../../../prebuilt_rules/logic/prepare_prebuilt_rules_for_import'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; import { createRulesAndExceptionsStreamFromNdJson } from '../../../logic/import/create_rules_stream_from_ndjson'; import { getReferencedExceptionLists } from '../../../logic/import/gather_referenced_exceptions'; @@ -104,13 +103,6 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C RuleExceptionsPromiseFromStreams[] >([request.body.file as HapiReadableStream, ...readAllStream]); - // TODO: optimize this so we only check/install if prebuilt rules are being imported? - if (prebuiltRulesCustomizationEnabled) { - const ruleAssetsClient = createPrebuiltRuleAssetsClient(savedObjectsClient); - await ensureLatestRulesPackageInstalled(ruleAssetsClient, config, ctx.securitySolution); - // TODO: calculate rule source, remove immutable field - } - // import exceptions, includes validation const { errors: exceptionsErrors, @@ -159,6 +151,13 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C savedObjectsClient, }); + const prebuiltRuleAssets = await preparePrebuiltRuleAssetsForImport({ + config, + context: ctx.securitySolution, + savedObjectsClient, + rules: parsedRules, + }); + const chunkParseObjects = chunk(CHUNK_PARSED_OBJECT_SIZE, parsedRules); const importRuleResponse: ImportRuleResponse[] = await importRulesHelper({ @@ -168,6 +167,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C detectionRulesClient, existingLists: foundReferencedExceptionLists, allowMissingConnectorSecrets: !!actionConnectors.length, + prebuiltRuleAssets, allowPrebuiltRules: prebuiltRulesCustomizationEnabled, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 1c45aeb1d3f76..6eebe6a82c96e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -18,6 +18,7 @@ import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; import { checkRuleExceptionReferences } from './check_rule_exception_references'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; +import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; export type PromiseFromStreams = RuleToImport | Error; export interface RuleExceptionsPromiseFromStreams { @@ -46,6 +47,7 @@ export const importRules = async ({ overwriteRules, detectionRulesClient, existingLists, + prebuiltRuleAssets, allowMissingConnectorSecrets, allowPrebuiltRules, }: { @@ -54,6 +56,7 @@ export const importRules = async ({ overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; existingLists: Record; + prebuiltRuleAssets?: PrebuiltRuleAsset[]; allowMissingConnectorSecrets?: boolean; allowPrebuiltRules?: boolean; }) => { @@ -101,6 +104,25 @@ export const importRules = async ({ return null; } + if (allowPrebuiltRules) { + if (parsedRule.rule_id && !parsedRule.version) { + resolve( + createBulkErrorObject({ + statusCode: 400, + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.cannotImportPrebuiltRuleWithoutVersion', + { + defaultMessage: 'Prebuilt rules must specify a "version" to be imported.', + } + ), + ruleId: parsedRule.rule_id, + }) + ); + } + + // TODO rule_source calculated here + } + try { const [exceptionErrors, exceptions] = checkRuleExceptionReferences({ rule: parsedRule, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index a160c5e70869b..a6ff5d998a095 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1624,13 +1624,29 @@ export default ({ getService }: FtrProviderContext): void => { error: { message: 'rule_id: Required', status_code: 400 }, }); }); - - it('installs prebuilt rules package if it is not installed'); }); describe('calculation of the rule_source fields', () => { it('calculates a version of 1 for custom rules'); - it('rejects a prebuilt rule with an unspecified version'); + + // TODO enable feature flag for this one + it.skip('rejects a prebuilt rule with an unspecified version', async () => { + const rule = getCustomQueryRuleParams(); + delete rule.version; + const ndjson = combineToNdJson(rule); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body.errors).toHaveLength(1); + expect(body.errors[0]).toMatchObject({ + error: { message: 'version: Required', status_code: 400 }, + }); + }); }); }); }); From a8b0a6dfa7faa3414b3f4d9ac600d9a349299ea4 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 13 Aug 2024 23:42:16 -0500 Subject: [PATCH 027/120] WIP: Implementing calculation of rule_source during import --- .../import/calculate_rule_source.test.ts | 79 +++++++++++++++++++ .../logic/import/calculate_rule_source.ts | 24 ++++++ .../logic/import/import_rules_utils.ts | 6 ++ 3 files changed, 109 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.test.ts new file mode 100644 index 0000000000000..8034cab515386 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { calculateRuleSource } from './calculate_rule_source'; + +describe('calculateRuleSource', () => { + it('should return internal type for custom rules', () => { + const rule = { rule_id: 'custom-rule', version: 1 }; + const prebuiltRuleAssets = []; + + const result = calculateRuleSource({ rule, prebuiltRuleAssets }); + + expect(result).toEqual({ type: 'internal' }); + }); + + it('should return external type for prebuilt rules with matching version', () => { + const rule = { rule_id: 'prebuilt-rule', version: 1 }; + const prebuiltRuleAssets = [ + { rule_id: 'prebuilt-rule', version: 1, source_updated_at: '2024-05-01' }, + ]; + + const result = calculateRuleSource({ rule, prebuiltRuleAssets }); + + expect(result).toEqual({ + type: 'external', + source_updated_at: '2024-05-01', + isCustomized: false, + }); + }); + + it('should return external type for prebuilt rules without matching version', () => { + const rule = { rule_id: 'prebuilt-rule', version: 999 }; + const prebuiltRuleAssets = [ + { rule_id: 'prebuilt-rule', version: 1, source_updated_at: '2024-05-01' }, + ]; + + const result = calculateRuleSource({ rule, prebuiltRuleAssets }); + + expect(result).toEqual({ + type: 'external', + isCustomized: false, + }); + }); + + it('should return external type with isCustomized true for prebuilt rules with customizations', () => { + const rule = { rule_id: 'prebuilt-rule', version: 1, name: 'Custom Prebuilt Rule' }; + const prebuiltRuleAssets = [ + { + rule_id: 'prebuilt-rule', + version: 1, + name: 'Prebuilt Rule', + source_updated_at: '2024-05-01', + }, + ]; + + const result = calculateRuleSource({ rule, prebuiltRuleAssets }); + + expect(result).toEqual({ + type: 'external', + source_updated_at: '2024-05-01', + isCustomized: true, + }); + }); + + it('should throw an error for prebuilt rules without version', () => { + const rule = { rule_id: 'prebuilt-rule' }; + const prebuiltRuleAssets = [ + { rule_id: 'prebuilt-rule', version: 1, source_updated_at: '2024-05-01' }, + ]; + + expect(() => calculateRuleSource({ rule, prebuiltRuleAssets })).toThrow( + 'version is required for prebuilt rules' + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.ts new file mode 100644 index 0000000000000..5822bd01e864f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleSource, RuleToImport } from '../../../../../../common/api/detection_engine'; +import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; + +/** + * Calculates the rule_source field for a rule being imported + * + * @param rule The rule to be imported + * @param prebuiltRuleAssets A list of prebuilt rule assets, which may include + * the installed version of the specified prebuilt rule. + */ +export const calculateRuleSource = ({ + rule, + prebuiltRuleAssets, +}: { + rule: RuleToImport; + prebuiltRuleAssets: PrebuiltRuleAsset[]; +}): RuleSource => {}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 6eebe6a82c96e..b8d8a0a54517e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -121,6 +121,12 @@ export const importRules = async ({ } // TODO rule_source calculated here + // const ruleSource = calculateRuleSource({ + // rule: parsedRule, + // prebuiltRuleAssets, + // }); + + // parsedRule.rule_source = ruleSource; } try { From 5ca0ab50f77177ec48366da0d015a87694bb34d0 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 14 Aug 2024 14:15:17 -0500 Subject: [PATCH 028/120] WIP: calculating rule source during import * Renames placeholder helper with more aptly-named one * Adds a common utility fn, calculateRuleSourceFromAsset, that I intend to leverage in the existing calculateRuleSource (if possible) * Adds tests for basic behavior There are lots type errors here; I think it's to do with the fact that we're dealing with both internal rule representations and request- and response-formatted rules. --- .../calculate_rule_source_from_asset.test.ts | 83 +++++++++++++++++++ .../calculate_rule_source_from_asset.ts | 41 +++++++++ .../import/calculate_rule_source.test.ts | 79 ------------------ .../logic/import/calculate_rule_source.ts | 24 ------ .../calculate_rule_source_for_import.test.ts | 71 ++++++++++++++++ .../calculate_rule_source_for_import.ts | 37 +++++++++ .../logic/import/import_rules_utils.ts | 12 +-- 7 files changed, 239 insertions(+), 108 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.test.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts new file mode 100644 index 0000000000000..31a096b295561 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../../../../../common/api/detection_engine'; +import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; +import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset'; + +const buildTestRule = (overrides?: Partial) => { + return { + rule_id: 'rule_id', + version: 1, + ...overrides, + } as RuleResponse; +}; + +const buildTestRuleAsset = (overrides?: Partial) => { + return { + rule_id: 'rule_id', + version: 1, + source_updated_at: '2024-05-01', + ...overrides, + } as PrebuiltRuleAsset; +}; + +describe('calculateRuleSourceFromAsset', () => { + it('calculates as internal if no asset is found', () => { + const result = calculateRuleSourceFromAsset({ + rule: buildTestRule(), + prebuiltRuleAsset: undefined, + ruleIdExists: false, + }); + + expect(result).toEqual({ + type: 'internal', + }); + }); + + it('calculates as unmodified external type if an asset is found without a matching version', () => { + const result = calculateRuleSourceFromAsset({ + rule: buildTestRule(), + prebuiltRuleAsset: buildTestRuleAsset({ version: 2 }), + ruleIdExists: true, + }); + + expect(result).toEqual({ + type: 'external', + is_customized: false, + }); + }); + + it('calculates as unmodified external type if an asset is not found but the rule ID exists', () => { + const result = calculateRuleSourceFromAsset({ + rule: buildTestRule(), + prebuiltRuleAsset: undefined, + ruleIdExists: true, + }); + + expect(result).toEqual({ + type: 'external', + is_customized: false, + }); + }); + + it('calculates as external with customizations if a matching asset/version is found', () => { + const rule = buildTestRule(); + + const result = calculateRuleSourceFromAsset({ + rule, + prebuiltRuleAsset: buildTestRuleAsset(), + ruleIdExists: true, + }); + + expect(result).toEqual({ + type: 'external', + source_updated_at: '2024-05-01', + is_customized: true, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts new file mode 100644 index 0000000000000..b71020719e836 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleSource, RuleResponse } from '../../../../../../../../common/api/detection_engine'; +import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; +import { calculateIsCustomized } from './calculate_is_customized'; + +export const calculateRuleSourceFromAsset = ({ + rule, + prebuiltRuleAsset, + ruleIdExists, +}: { + rule: RuleResponse; + prebuiltRuleAsset: PrebuiltRuleAsset | undefined; + ruleIdExists: boolean; +}): RuleSource => { + if (!ruleIdExists) { + return { + type: 'internal', + }; + } + + if (prebuiltRuleAsset == null) { + return { + type: 'external', + is_customized: false, + }; + } + + const isCustomized = calculateIsCustomized(prebuiltRuleAsset, rule); + + return { + type: 'external', + is_customized: isCustomized, + source_updated_at: prebuiltRuleAsset.source_updated_at, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.test.ts deleted file mode 100644 index 8034cab515386..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { calculateRuleSource } from './calculate_rule_source'; - -describe('calculateRuleSource', () => { - it('should return internal type for custom rules', () => { - const rule = { rule_id: 'custom-rule', version: 1 }; - const prebuiltRuleAssets = []; - - const result = calculateRuleSource({ rule, prebuiltRuleAssets }); - - expect(result).toEqual({ type: 'internal' }); - }); - - it('should return external type for prebuilt rules with matching version', () => { - const rule = { rule_id: 'prebuilt-rule', version: 1 }; - const prebuiltRuleAssets = [ - { rule_id: 'prebuilt-rule', version: 1, source_updated_at: '2024-05-01' }, - ]; - - const result = calculateRuleSource({ rule, prebuiltRuleAssets }); - - expect(result).toEqual({ - type: 'external', - source_updated_at: '2024-05-01', - isCustomized: false, - }); - }); - - it('should return external type for prebuilt rules without matching version', () => { - const rule = { rule_id: 'prebuilt-rule', version: 999 }; - const prebuiltRuleAssets = [ - { rule_id: 'prebuilt-rule', version: 1, source_updated_at: '2024-05-01' }, - ]; - - const result = calculateRuleSource({ rule, prebuiltRuleAssets }); - - expect(result).toEqual({ - type: 'external', - isCustomized: false, - }); - }); - - it('should return external type with isCustomized true for prebuilt rules with customizations', () => { - const rule = { rule_id: 'prebuilt-rule', version: 1, name: 'Custom Prebuilt Rule' }; - const prebuiltRuleAssets = [ - { - rule_id: 'prebuilt-rule', - version: 1, - name: 'Prebuilt Rule', - source_updated_at: '2024-05-01', - }, - ]; - - const result = calculateRuleSource({ rule, prebuiltRuleAssets }); - - expect(result).toEqual({ - type: 'external', - source_updated_at: '2024-05-01', - isCustomized: true, - }); - }); - - it('should throw an error for prebuilt rules without version', () => { - const rule = { rule_id: 'prebuilt-rule' }; - const prebuiltRuleAssets = [ - { rule_id: 'prebuilt-rule', version: 1, source_updated_at: '2024-05-01' }, - ]; - - expect(() => calculateRuleSource({ rule, prebuiltRuleAssets })).toThrow( - 'version is required for prebuilt rules' - ); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.ts deleted file mode 100644 index 5822bd01e864f..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { RuleSource, RuleToImport } from '../../../../../../common/api/detection_engine'; -import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; - -/** - * Calculates the rule_source field for a rule being imported - * - * @param rule The rule to be imported - * @param prebuiltRuleAssets A list of prebuilt rule assets, which may include - * the installed version of the specified prebuilt rule. - */ -export const calculateRuleSource = ({ - rule, - prebuiltRuleAssets, -}: { - rule: RuleToImport; - prebuiltRuleAssets: PrebuiltRuleAsset[]; -}): RuleSource => {}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts new file mode 100644 index 0000000000000..8200834c98921 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../../../common/api/detection_engine'; +import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; +import { calculateRuleSourceForImport } from './calculate_rule_source_for_import'; + +const buildTestRule = (overrides?: Partial) => { + return { + rule_id: 'rule_id', + version: 1, + ...overrides, + } as RuleResponse; +}; + +const buildTestRuleAsset = (overrides?: Partial) => { + return { + rule_id: 'rule_id', + version: 1, + source_updated_at: '2024-05-01', + ...overrides, + } as PrebuiltRuleAsset; +}; + +describe('calculateRuleSourceForImport', () => { + it('calculates as internal if no asset is found', () => { + const result = calculateRuleSourceForImport({ + rule: buildTestRule(), + prebuiltRuleAssets: [], + installedRuleIds: [], + }); + + expect(result).toEqual({ + type: 'internal', + }); + }); + + it('calculates as unmodified external type if an asset is found without a matching version', () => { + const result = calculateRuleSourceForImport({ + rule: buildTestRule(), + prebuiltRuleAssets: [], + installedRuleIds: ['rule_id'], + }); + + expect(result).toEqual({ + type: 'external', + isCustomized: false, + }); + }); + + it('calculates as external with customizations if a matching asset/version is found', () => { + const rule = buildTestRule(); + const prebuiltRuleAssets = [buildTestRuleAsset()]; + + const result = calculateRuleSourceForImport({ + rule, + prebuiltRuleAssets, + installedRuleIds: ['rule_id'], + }); + + expect(result).toEqual({ + type: 'external', + source_updated_at: '2024-05-01', + isCustomized: true, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts new file mode 100644 index 0000000000000..a4cc124efb28c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleSource, RuleResponse } from '../../../../../../common/api/detection_engine'; +import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; +import { calculateRuleSourceFromAsset } from '../detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset'; + +/** + * Calculates the rule_source field for a rule being imported + * + * @param rule The rule to be imported + * @param prebuiltRuleAssets A list of prebuilt rule assets, which may include + * the installed version of the specified prebuilt rule. + * @param installedRuleIds A list of prebuilt rule IDs that are currently installed + */ +export const calculateRuleSourceForImport = ({ + rule, + prebuiltRuleAssets, + installedRuleIds, +}: { + rule: RuleResponse; + prebuiltRuleAssets: PrebuiltRuleAsset[]; + installedRuleIds: string[]; +}): RuleSource => { + const matchingAsset = prebuiltRuleAssets.find((asset) => asset.rule_id === rule.rule_id); + const ruleIdExists = installedRuleIds.includes(rule.rule_id); + + return calculateRuleSourceFromAsset({ + rule, + prebuiltRuleAsset: matchingAsset, + ruleIdExists, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index b8d8a0a54517e..10edb1fae623c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -17,6 +17,7 @@ import type { RuleToImport } from '../../../../../../common/api/detection_engine import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; import { checkRuleExceptionReferences } from './check_rule_exception_references'; +import { calculateRuleSourceForImport } from './calculate_rule_source_for_import'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; @@ -121,12 +122,13 @@ export const importRules = async ({ } // TODO rule_source calculated here - // const ruleSource = calculateRuleSource({ - // rule: parsedRule, - // prebuiltRuleAssets, - // }); + const ruleSource = calculateRuleSourceForImport({ + rule: parsedRule, + prebuiltRuleAssets, + installedRuleIds: [], // TODO + }); - // parsedRule.rule_source = ruleSource; + parsedRule.rule_source = ruleSource; } try { From db19214490ce7dd5ab3a546e6b616286a9335a80 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 19 Aug 2024 14:07:45 -0500 Subject: [PATCH 029/120] Add a new type representing the import of a prebuilt rule --- .../rule_management/import_rules/rule_to_import.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index f7ab252171667..ae20bc3fb46c2 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -12,6 +12,7 @@ import { RequiredFieldInput, RuleSignatureId, TypeSpecificCreateProps, + RuleVersion, } from '../../model/rule_schema'; /** @@ -40,3 +41,12 @@ export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( required_fields: z.array(RequiredFieldInput).optional(), }) ); + +/** + * PrebuiltRuleToImport represents a RuleToImport that has been validated as + * representing a prebuilt rule. Part of this definition is the inclusion of a + * "version" specifier, which is represented here. + */ +export type PrebuiltRuleToImport = z.infer; +export type PrebuiltRuleToImportInput = z.input; +export const PrebuiltRuleToImport = RuleToImport.and(z.object({ version: RuleVersion })); From dc57d510cf3584cf8400a832045e6fa221ab2760 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 19 Aug 2024 14:23:24 -0500 Subject: [PATCH 030/120] Fixes type issues when calculating rule source for import * Allows diff utilities to receive a `PrebuiltRuleImport` * Fixes types at call site of import by adding a type guard I did add a TODO about distinguishing the non-prebuilt case during import; I need to go back to the RFC to answer that, I think. --- .../normalization/convert_rule_to_diffable.ts | 7 +++++-- .../rule_source/calculate_is_customized.ts | 7 +++++-- .../calculate_rule_source_from_asset.ts | 8 ++++++-- .../import/calculate_rule_source_for_import.ts | 7 +++++-- .../logic/import/import_rules_utils.ts | 15 +++++++++++---- 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts index 525af83d48b99..86dfd57158a21 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts @@ -9,6 +9,7 @@ import type { RequiredOptional } from '@kbn/zod-helpers'; import { requiredOptional } from '@kbn/zod-helpers'; import { DEFAULT_MAX_SIGNALS } from '../../../../../../../common/constants'; import { assertUnreachable } from '../../../../../../../common/utility_types'; +import type { PrebuiltRuleToImport } from '../../../../../../../common/api/detection_engine'; import type { EqlRule, EqlRuleCreateProps, @@ -59,7 +60,9 @@ import { addEcsToRequiredFields } from '../../../../rule_management/utils/utils' * Normalizes a given rule to the form which is suitable for passing to the diff algorithm. * Read more in the JSDoc description of DiffableRule. */ -export const convertRuleToDiffable = (rule: RuleResponse | PrebuiltRuleAsset): DiffableRule => { +export const convertRuleToDiffable = ( + rule: RuleResponse | PrebuiltRuleAsset | PrebuiltRuleToImport +): DiffableRule => { const commonFields = extractDiffableCommonFields(rule); switch (rule.type) { @@ -109,7 +112,7 @@ export const convertRuleToDiffable = (rule: RuleResponse | PrebuiltRuleAsset): D }; const extractDiffableCommonFields = ( - rule: RuleResponse | PrebuiltRuleAsset + rule: RuleResponse | PrebuiltRuleAsset | PrebuiltRuleToImport ): RequiredOptional => { return { // --------------------- REQUIRED FIELDS diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts index 4f9bb4a060f6f..8ea85eda78e17 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts @@ -5,7 +5,10 @@ * 2.0. */ -import type { RuleResponse } from '../../../../../../../../common/api/detection_engine'; +import type { + PrebuiltRuleToImport, + RuleResponse, +} from '../../../../../../../../common/api/detection_engine'; import { MissingVersion } from '../../../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateRuleFieldsDiff } from '../../../../../prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff'; @@ -14,7 +17,7 @@ import { convertPrebuiltRuleAssetToRuleResponse } from '../../converters/convert export function calculateIsCustomized( baseRule: PrebuiltRuleAsset | undefined, - nextRule: RuleResponse + nextRule: RuleResponse | PrebuiltRuleToImport ) { if (baseRule == null) { // If the base version is missing, we consider the rule to be customized diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts index b71020719e836..5da267d8b289b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts @@ -5,7 +5,11 @@ * 2.0. */ -import type { RuleSource, RuleResponse } from '../../../../../../../../common/api/detection_engine'; +import type { + RuleSource, + RuleResponse, + PrebuiltRuleToImport, +} from '../../../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateIsCustomized } from './calculate_is_customized'; @@ -14,7 +18,7 @@ export const calculateRuleSourceFromAsset = ({ prebuiltRuleAsset, ruleIdExists, }: { - rule: RuleResponse; + rule: RuleResponse | PrebuiltRuleToImport; prebuiltRuleAsset: PrebuiltRuleAsset | undefined; ruleIdExists: boolean; }): RuleSource => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index a4cc124efb28c..5d7f5db991afa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -5,7 +5,10 @@ * 2.0. */ -import type { RuleSource, RuleResponse } from '../../../../../../common/api/detection_engine'; +import type { + PrebuiltRuleToImport, + RuleSource, +} from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; import { calculateRuleSourceFromAsset } from '../detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset'; @@ -22,7 +25,7 @@ export const calculateRuleSourceForImport = ({ prebuiltRuleAssets, installedRuleIds, }: { - rule: RuleResponse; + rule: PrebuiltRuleToImport; prebuiltRuleAssets: PrebuiltRuleAsset[]; installedRuleIds: string[]; }): RuleSource => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 10edb1fae623c..071b8a6570206 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -13,7 +13,10 @@ import type { ExceptionListSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; +import type { + PrebuiltRuleToImport, + RuleToImport, +} from '../../../../../../common/api/detection_engine/rule_management'; import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; import { checkRuleExceptionReferences } from './check_rule_exception_references'; @@ -106,7 +109,8 @@ export const importRules = async ({ } if (allowPrebuiltRules) { - if (parsedRule.rule_id && !parsedRule.version) { + // TODO how do we differentiate this case from a non-prebuilt import? + if (!ruleImportIsPrebuilt(parsedRule)) { resolve( createBulkErrorObject({ statusCode: 400, @@ -119,12 +123,13 @@ export const importRules = async ({ ruleId: parsedRule.rule_id, }) ); + + return null; } - // TODO rule_source calculated here const ruleSource = calculateRuleSourceForImport({ rule: parsedRule, - prebuiltRuleAssets, + prebuiltRuleAssets: prebuiltRuleAssets ?? [], installedRuleIds: [], // TODO }); @@ -174,3 +179,5 @@ export const importRules = async ({ return importRuleResponse; }; + +const ruleImportIsPrebuilt = (rule: RuleToImport): rule is PrebuiltRuleToImport => !!rule.version; From 9e887b0713cf4409ea7316f4208735eaf787a881 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 19 Aug 2024 23:02:23 -0500 Subject: [PATCH 031/120] Refactor prebuilt import logic similar to #190447 This splits our old utility, `preparePrebuiltRulesForImport`, into a class that has to responsibilities: 1. Ensures that the latest prebuilt #ssets are retrieved during `setup` 2. Retrieves the relevant prebuilt assets from a list of rules to import Similar to #190447, this was done so that we perform the per-rule work in chunks during import, rather than all at once beforehand. --- .../logic/prebuilt_rules_importer.test.ts | 110 ++++++++++++++++++ .../logic/prebuilt_rules_importer.ts | 96 +++++++++++++++ .../prepare_prebuilt_rules_for_import.ts | 54 --------- .../api/rules/import_rules/route.ts | 7 +- .../logic/import/import_rules_utils.ts | 15 ++- 5 files changed, 220 insertions(+), 62 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prepare_prebuilt_rules_for_import.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts new file mode 100644 index 0000000000000..85a37a922319a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from '@kbn/core/server'; +import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import type { RuleToImport } from '../../../../../common/api/detection_engine'; +import { createPrebuiltRuleAssetsClient as createPrebuiltRuleAssetsClientMock } from './rule_assets/__mocks__/prebuilt_rule_assets_client'; +import { ensureLatestRulesPackageInstalled } from './ensure_latest_rules_package_installed'; +import { configMock, createMockConfig, requestContextMock } from '../../routes/__mocks__'; +import { PrebuiltRulesImporter } from './prebuilt_rules_importer'; + +jest.mock('./ensure_latest_rules_package_installed'); + +let mockPrebuiltRuleAssetsClient: ReturnType; + +jest.mock('./rule_assets/prebuilt_rule_assets_client', () => ({ + createPrebuiltRuleAssetsClient: () => mockPrebuiltRuleAssetsClient, +})); + +describe('PrebuiltRulesImporter', () => { + let config: ReturnType; + let context: ReturnType['securitySolution']; + let savedObjectsClient: jest.Mocked; + + beforeEach(() => { + config = createMockConfig(); + config = configMock.withExperimentalFeature(config, 'prebuiltRulesCustomizationEnabled'); + context = requestContextMock.create().securitySolution; + savedObjectsClient = savedObjectsClientMock.create(); + mockPrebuiltRuleAssetsClient = createPrebuiltRuleAssetsClientMock(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should initialize correctly', () => { + const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + + expect(importer).toBeDefined(); + expect(importer.enabled).toBe(true); + expect(importer.latestPackagesInstalled).toBe(false); + }); + + describe('setup', () => { + it('should not call ensureLatestRulesPackageInstalled if disabled', async () => { + config = createMockConfig(); + const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + + await importer.setup(); + + expect(ensureLatestRulesPackageInstalled).not.toHaveBeenCalled(); + expect(importer.latestPackagesInstalled).toBe(false); + }); + + it('should call ensureLatestRulesPackageInstalled if enabled', async () => { + const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + + await importer.setup(); + + expect(ensureLatestRulesPackageInstalled).toHaveBeenCalledWith( + mockPrebuiltRuleAssetsClient, + config, + context + ); + expect(importer.latestPackagesInstalled).toBe(true); + }); + }); + + describe('fetchPrebuiltRuleAssets', () => { + it('should return an empty array if disabled', async () => { + config = createMockConfig(); + const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + + const result = await importer.fetchPrebuiltRuleAssets({ rules: [] }); + + expect(result).toEqual([]); + }); + + it('should throw an error if latestPackagesInstalled is false', async () => { + const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + + await expect(importer.fetchPrebuiltRuleAssets({ rules: [] })).rejects.toThrow( + 'Prebuilt rule assets cannot be fetched until the latest rules package is installed. Call setup() on this object first.' + ); + }); + + it('should fetch prebuilt rule assets correctly if latestPackagesInstalled is true', async () => { + const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + importer.latestPackagesInstalled = true; + + const rules = [ + { rule_id: 'rule-1', version: 1 }, + { rule_id: 'rule-2', version: 2 }, + new Error('Invalid rule'), + ] as Array; + + await importer.fetchPrebuiltRuleAssets({ rules }); + + expect(mockPrebuiltRuleAssetsClient.fetchAssetsByVersion).toHaveBeenCalledWith([ + { rule_id: 'rule-1', version: 1 }, + { rule_id: 'rule-2', version: 2 }, + ]); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts new file mode 100644 index 0000000000000..fc917ef932a9d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from '@kbn/core/server'; +import type { RuleToImport } from '../../../../../common/api/detection_engine'; +import type { SecuritySolutionApiRequestHandlerContext } from '../../../../types'; +import type { ConfigType } from '../../../../config'; +import type { PrebuiltRuleAsset } from '../model/rule_assets/prebuilt_rule_asset'; +import { createPrebuiltRuleAssetsClient } from './rule_assets/prebuilt_rule_assets_client'; +import { ensureLatestRulesPackageInstalled } from './ensure_latest_rules_package_installed'; + +type MaybeRule = RuleToImport | Error; +export interface IPrebuiltRulesImporter { + setup: () => Promise; + fetchPrebuiltRuleAssets: ({ rules }: { rules: MaybeRule[] }) => Promise; +} + +/** + * + * This object contains logic necessary to import prebuilt rules into the system. + */ +export class PrebuiltRulesImporter implements IPrebuiltRulesImporter { + private context: SecuritySolutionApiRequestHandlerContext; + private config: ConfigType; + private ruleAssetsClient: ReturnType; + public enabled: boolean; + public latestPackagesInstalled: boolean = false; + + constructor({ + config, + savedObjectsClient, + context, + }: { + config: ConfigType; + context: SecuritySolutionApiRequestHandlerContext; + savedObjectsClient: SavedObjectsClientContract; + }) { + this.ruleAssetsClient = createPrebuiltRuleAssetsClient(savedObjectsClient); + this.context = context; + this.config = config; + this.enabled = config.experimentalFeatures.prebuiltRulesCustomizationEnabled; + } + + /** + * + * Prepares the system to import prebuilt rules by ensuring the latest rules + * package is installed. + */ + public async setup() { + if (!this.enabled) { + return; + } + + await ensureLatestRulesPackageInstalled(this.ruleAssetsClient, this.config, this.context); + this.latestPackagesInstalled = true; + } + + /** + * Retrieves the prebuilt rule assets for the specified rules being imported. If + * @param rules - The rules to be imported + * + * @returns The prebuilt rule assets corresponding to the specified prebuilt + * rules, which are used to determine how to import those rules (create vs. update, etc.). + */ + public async fetchPrebuiltRuleAssets({ + rules, + }: { + rules: MaybeRule[]; + }): Promise { + if (!this.enabled) { + return []; + } + + if (!this.latestPackagesInstalled) { + throw new Error( + 'Prebuilt rule assets cannot be fetched until the latest rules package is installed. Call setup() on this object first.' + ); + } + + const prebuiltRulesToImport = rules.flatMap((rule) => { + if (rule instanceof Error || rule.version == null) { + return []; + } + return { + rule_id: rule.rule_id, + version: rule.version, + }; + }); + + return this.ruleAssetsClient.fetchAssetsByVersion(prebuiltRulesToImport); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prepare_prebuilt_rules_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prepare_prebuilt_rules_for_import.ts deleted file mode 100644 index ec739dfe9bac9..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prepare_prebuilt_rules_for_import.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { SavedObjectsClientContract } from '@kbn/core/server'; -import type { SecuritySolutionApiRequestHandlerContext } from '../../../../types'; -import type { RuleToImport } from '../../../../../common/api/detection_engine'; -import type { ConfigType } from '../../../../config'; -import type { PrebuiltRuleAsset } from '../model/rule_assets/prebuilt_rule_asset'; -import { createPrebuiltRuleAssetsClient } from './rule_assets/prebuilt_rule_assets_client'; -import { ensureLatestRulesPackageInstalled } from './ensure_latest_rules_package_installed'; - -/** - * - * Prepares the system to import prebuilt rules by ensuring the latest rules - * package is installed and fetching the corresponding prebuilt rule assets from the package. - * @param rules - The rules to be imported - * - * @returns The prebuilt rule assets corresponding to the specified prebuilt - * rules, which are used to determine how to import those rules (create vs. update, etc.). - */ -export const preparePrebuiltRuleAssetsForImport = async ({ - savedObjectsClient, - config, - context, - rules, -}: { - config: ConfigType; - context: SecuritySolutionApiRequestHandlerContext; - savedObjectsClient: SavedObjectsClientContract; - rules: Array; -}): Promise => { - if (!config.experimentalFeatures.prebuiltRulesCustomizationEnabled) { - return []; - } - - const ruleAssetsClient = createPrebuiltRuleAssetsClient(savedObjectsClient); - await ensureLatestRulesPackageInstalled(ruleAssetsClient, config, context); - - const prebuiltRulesToImport = rules.flatMap((rule) => { - if (rule instanceof Error || rule.version == null) { - return []; - } - return { - rule_id: rule.rule_id, - version: rule.version, - }; - }); - - return ruleAssetsClient.fetchAssetsByVersion(prebuiltRulesToImport); -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index c3493155c08ac..0fca6a104f13c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -21,7 +21,7 @@ import type { ConfigType } from '../../../../../../config'; import type { HapiReadableStream, SecuritySolutionPluginRouter } from '../../../../../../types'; import type { BulkError, ImportRuleResponse } from '../../../../routes/utils'; import { buildSiemResponse, isBulkError, isImportRegular } from '../../../../routes/utils'; -import { preparePrebuiltRuleAssetsForImport } from '../../../../prebuilt_rules/logic/prepare_prebuilt_rules_for_import'; +import { PrebuiltRulesImporter } from '../../../../prebuilt_rules/logic/prebuilt_rules_importer'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; import { createRulesAndExceptionsStreamFromNdJson } from '../../../logic/import/create_rules_stream_from_ndjson'; import type { RuleExceptionsPromiseFromStreams } from '../../../logic/import/import_rules_utils'; @@ -144,11 +144,10 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C ? [] : rulesWithMigratedActions || migratedParsedObjectsWithoutDuplicateErrors; - const prebuiltRuleAssets = await preparePrebuiltRuleAssetsForImport({ + const prebuiltRulesImporter = new PrebuiltRulesImporter({ config, context: ctx.securitySolution, savedObjectsClient, - rules: parsedRules, }); const chunkParseObjects = chunk(CHUNK_PARSED_OBJECT_SIZE, parsedRules); @@ -159,7 +158,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C overwriteRules: request.query.overwrite, detectionRulesClient, allowMissingConnectorSecrets: !!actionConnectors.length, - prebuiltRuleAssets, + prebuiltRulesImporter, allowPrebuiltRules: prebuiltRulesCustomizationEnabled, savedObjectsClient, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index b2dcd6d2f66ca..acd69b4056a61 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -18,10 +18,10 @@ import type { } from '../../../../../../common/api/detection_engine/rule_management'; import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; +import type { PrebuiltRulesImporter } from '../../../prebuilt_rules/logic/prebuilt_rules_importer'; import { checkRuleExceptionReferences } from './check_rule_exception_references'; import { calculateRuleSourceForImport } from './calculate_rule_source_for_import'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; -import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; import { getReferencedExceptionLists } from './gather_referenced_exceptions'; export type PromiseFromStreams = RuleToImport | Error; @@ -50,7 +50,7 @@ export const importRules = async ({ rulesResponseAcc, overwriteRules, detectionRulesClient, - prebuiltRuleAssets, + prebuiltRulesImporter, allowPrebuiltRules, allowMissingConnectorSecrets, savedObjectsClient, @@ -59,7 +59,7 @@ export const importRules = async ({ rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; - prebuiltRuleAssets?: PrebuiltRuleAsset[]; + prebuiltRulesImporter: PrebuiltRulesImporter; allowPrebuiltRules?: boolean; allowMissingConnectorSecrets?: boolean; savedObjectsClient: SavedObjectsClientContract; @@ -72,12 +72,19 @@ export const importRules = async ({ return importRuleResponse; } + // TODO only call this if prebuilt rules are being imported? + prebuiltRulesImporter.setup(); + while (ruleChunks.length) { const batchParseObjects = ruleChunks.shift() ?? []; const existingLists = await getReferencedExceptionLists({ rules: batchParseObjects, savedObjectsClient, }); + const prebuiltRuleAssets = await prebuiltRulesImporter.fetchPrebuiltRuleAssets({ + rules: batchParseObjects, + }); + const newImportRuleResponse = await Promise.all( batchParseObjects.reduce>>((accum, parsedRule) => { const importsWorkerPromise = new Promise(async (resolve, reject) => { @@ -133,7 +140,7 @@ export const importRules = async ({ const ruleSource = calculateRuleSourceForImport({ rule: parsedRule, - prebuiltRuleAssets: prebuiltRuleAssets ?? [], + prebuiltRuleAssets, installedRuleIds: [], // TODO }); From ff01984d0a6a6dc57252bfabc6950d3e8443fc7f Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 19 Aug 2024 23:34:35 -0500 Subject: [PATCH 032/120] Fixes import_rules_utils tests * Adds a mock file for our new class * Updates existing tests that failed due to API changes * Adds a new case for a logical branch in our utility The "success" test is currently failing because we're actually calling `calculateRuleSourceForImport`, but that's currently expected. --- .../logic/prebuilt_rules_importer.mock.ts | 18 +++++++++ .../logic/import/import_rules_utils.test.ts | 39 ++++++++++++++++++- 2 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts new file mode 100644 index 0000000000000..43490b1acbe0e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PrebuiltRulesImporter } from './prebuilt_rules_importer'; + +const createPrebuiltRulesImporterMock = (): jest.Mocked => + ({ + setup: jest.fn(), + fetchPrebuiltRuleAssets: jest.fn(), + } as unknown as jest.Mocked); + +export const prebuiltRulesImporterMock = { + create: createPrebuiltRulesImporterMock, +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts index 7f6ba2163d91c..d78d899ebafdd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts @@ -10,6 +10,7 @@ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; import { requestContextMock } from '../../../routes/__mocks__'; +import { prebuiltRulesImporterMock } from '../../../prebuilt_rules/logic/prebuilt_rules_importer.mock'; import { importRules } from './import_rules_utils'; import { createBulkErrorObject } from '../../../routes/utils'; @@ -19,11 +20,13 @@ describe('importRules', () => { const ruleToImport = getImportRulesSchemaMock(); let savedObjectsClient: jest.Mocked; + let mockPrebuiltRulesImporter: ReturnType; beforeEach(() => { jest.clearAllMocks(); savedObjectsClient = savedObjectsClientMock.create(); + mockPrebuiltRulesImporter = prebuiltRulesImporterMock.create(); }); it('returns an empty rules response if no rules to import', async () => { @@ -33,6 +36,7 @@ describe('importRules', () => { overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), savedObjectsClient, + prebuiltRulesImporter: mockPrebuiltRulesImporter, }); expect(result).toEqual([]); @@ -44,6 +48,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + prebuiltRulesImporter: mockPrebuiltRulesImporter, savedObjectsClient, }); @@ -73,6 +78,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + prebuiltRulesImporter: mockPrebuiltRulesImporter, savedObjectsClient, }); @@ -99,6 +105,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + prebuiltRulesImporter: mockPrebuiltRulesImporter, savedObjectsClient, }); @@ -112,6 +119,7 @@ describe('importRules', () => { prebuiltRuleToImport = { ...getImportRulesSchemaMock(), immutable: true, + version: 1, }; }); @@ -122,8 +130,34 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - existingLists: {}, + prebuiltRulesImporter: mockPrebuiltRulesImporter, allowPrebuiltRules: false, + savedObjectsClient, + }); + + expect(result).toEqual([ + { + error: { + message: + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, remove its "immutable" property try again.', + status_code: 400, + }, + rule_id: prebuiltRuleToImport.rule_id, + }, + ]); + }); + + it('rejects a prebuilt rule when version is missing', async () => { + delete prebuiltRuleToImport.version; + const ruleChunk = [prebuiltRuleToImport]; + const result = await importRules({ + ruleChunks: [ruleChunk], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + prebuiltRulesImporter: mockPrebuiltRulesImporter, + allowPrebuiltRules: false, + savedObjectsClient, }); expect(result).toEqual([ @@ -150,8 +184,9 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - existingLists: {}, allowPrebuiltRules: true, + prebuiltRulesImporter: mockPrebuiltRulesImporter, + savedObjectsClient, }); expect(result).toEqual([{ rule_id: prebuiltRuleToImport.rule_id, status_code: 200 }]); From aa1e6f7673ffe02b68d60043c9be458897e7f328 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 22 Aug 2024 15:02:36 -0500 Subject: [PATCH 033/120] Remove TODO We can't know whether the rules being imported are prebuilt or not until we compare the `rule_id/version` to our list of assets. --- .../rule_management/logic/import/import_rules_utils.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index acd69b4056a61..b9ee17f48360a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -72,7 +72,6 @@ export const importRules = async ({ return importRuleResponse; } - // TODO only call this if prebuilt rules are being imported? prebuiltRulesImporter.setup(); while (ruleChunks.length) { From b88dac31a80894848889cc46f0ac11ffd675695d Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 22 Aug 2024 16:55:08 -0500 Subject: [PATCH 034/120] Add logic for retrieving installed prebuilt rules by rule_id We need this in the case where the rule being imported is not found by both `rule_id` and `version` (using `fetchAssetsByVersion`); at that point we know that the rule cannot be compared to an existing version, but we need this additional information to determine whether the rule represents a prebuilt rule (i.e. a known `rule_id`) or a custom rule (no known `rule_id`). --- .../logic/prebuilt_rules_importer.mock.ts | 1 + .../logic/prebuilt_rules_importer.test.ts | 56 +++++++++++++++++++ .../logic/prebuilt_rules_importer.ts | 33 +++++++++++ .../__mocks__/prebuilt_rule_assets_client.ts | 1 + .../prebuilt_rule_assets_client.ts | 55 ++++++++++++++++++ .../logic/import/import_rules_utils.ts | 5 +- 6 files changed, 150 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts index 43490b1acbe0e..d96d0bc0977eb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts @@ -11,6 +11,7 @@ const createPrebuiltRulesImporterMock = (): jest.Mocked = ({ setup: jest.fn(), fetchPrebuiltRuleAssets: jest.fn(), + fetchInstalledRuleIds: jest.fn(), } as unknown as jest.Mocked); export const prebuiltRulesImporterMock = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts index 85a37a922319a..23405a1609cde 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts @@ -107,4 +107,60 @@ describe('PrebuiltRulesImporter', () => { ]); }); }); + + describe('fetchInstalledRuleIds', () => { + it('returns an empty array if the importer is not enabled', async () => { + const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + importer.enabled = false; + + const result = await importer.fetchInstalledRuleIds({ rules: [] }); + + expect(result).toEqual([]); + }); + + it('throws an error if the latest packages are not installed', async () => { + const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + importer.latestPackagesInstalled = false; + + await expect(importer.fetchInstalledRuleIds({ rules: [] })).rejects.toThrow( + 'Installed rule IDs cannot be fetched until the latest rules package is installed. Call setup() on this object first.' + ); + }); + + it('fetches and return the rule IDs of installed prebuilt rules', async () => { + const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + importer.latestPackagesInstalled = true; + const rules = [ + { rule_id: 'rule-1', version: 1 }, + { rule_id: 'rule-2', version: 1 }, + new Error('Invalid rule'), + ] as Array; + const installedRuleAssets = [{ rule_id: 'rule-1' }, { rule_id: 'rule-2' }]; + + (mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId as jest.Mock).mockResolvedValue( + installedRuleAssets + ); + + const result = await importer.fetchInstalledRuleIds({ rules }); + + expect(mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId).toHaveBeenCalledWith([ + 'rule-1', + 'rule-2', + ]); + expect(result).toEqual(['rule-1', 'rule-2']); + }); + + it('handles rules that are instances of Error', async () => { + const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + importer.latestPackagesInstalled = true; + const rules = [new Error('Invalid rule')]; + + (mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId as jest.Mock).mockResolvedValue([]); + + const result = await importer.fetchInstalledRuleIds({ rules }); + + expect(mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId).toHaveBeenCalledWith([]); + expect(result).toEqual([]); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts index fc917ef932a9d..a4e08353ba87b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts @@ -93,4 +93,37 @@ export class PrebuiltRulesImporter implements IPrebuiltRulesImporter { return this.ruleAssetsClient.fetchAssetsByVersion(prebuiltRulesToImport); } + + /** + * Retrieves the rule IDs (`rule_id`) of installed prebuilt rules matching those + * of the specified rules. Can be used to determine whether the rule being + * imported is a custom rule or a prebuilt rule. + * + * @param rules - A list of rules being imported. + * + * @returns A list of the rule IDs that are installed. + * + */ + public async fetchInstalledRuleIds({ rules }: { rules: MaybeRule[] }): Promise { + if (!this.enabled) { + return []; + } + + if (!this.latestPackagesInstalled) { + throw new Error( + 'Installed rule IDs cannot be fetched until the latest rules package is installed. Call setup() on this object first.' + ); + } + + const ruleIds = rules.flatMap((rule) => { + if (rule instanceof Error) { + return []; + } + return rule.rule_id; + }); + + const installedRuleAssets = await this.ruleAssetsClient.fetchLatestAssetsByRuleId(ruleIds); + + return installedRuleAssets.map((asset) => asset.rule_id); + } } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client.ts index 0776fefb98656..b9233a121ccf6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client.ts @@ -8,6 +8,7 @@ export const createPrebuiltRuleAssetsClient = () => { return { fetchLatestAssets: jest.fn(), + fetchLatestAssetsByRuleId: jest.fn(), fetchLatestVersions: jest.fn(), fetchAssetsByVersion: jest.fn(), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts index 3daaab8ecf10f..fd0fdac62cdbf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts @@ -20,6 +20,8 @@ const MAX_PREBUILT_RULES_COUNT = 10_000; export interface IPrebuiltRuleAssetsClient { fetchLatestAssets: () => Promise; + fetchLatestAssetsByRuleId(ruleIds: string[]): Promise; + fetchLatestVersions(): Promise; fetchAssetsByVersion(versions: RuleVersionSpecifier[]): Promise; @@ -72,6 +74,59 @@ export const createPrebuiltRuleAssetsClient = ( }); }, + fetchLatestAssetsByRuleId: (ruleIds: string[]) => { + return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId', async () => { + if (ruleIds.length === 0) { + // NOTE: without early return it would build incorrect filter and fetch all existing saved objects + return []; + } + + const filter = ruleIds + .map((ruleId) => `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id: ${ruleId}`) + .join(' OR '); + + const findResult = await savedObjectsClient.find< + PrebuiltRuleAsset, + { + rules: AggregationsMultiBucketAggregateBase<{ + latest_version: AggregationsTopHitsAggregate; + }>; + } + >({ + type: PREBUILT_RULE_ASSETS_SO_TYPE, + filter, + aggs: { + rules: { + terms: { + field: `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id`, + size: MAX_PREBUILT_RULES_COUNT, + }, + aggs: { + latest_version: { + top_hits: { + size: 1, + sort: { + [`${PREBUILT_RULE_ASSETS_SO_TYPE}.version`]: 'desc', + }, + }, + }, + }, + }, + }, + }); + + const buckets = findResult.aggregations?.rules?.buckets ?? []; + invariant(Array.isArray(buckets), 'Expected buckets to be an array'); + + const ruleAssets = buckets.map((bucket) => { + const hit = bucket.latest_version.hits.hits[0]; + return hit._source[PREBUILT_RULE_ASSETS_SO_TYPE]; + }); + + return validatePrebuiltRuleAssets(ruleAssets); + }); + }, + fetchLatestVersions: (): Promise => { return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchLatestVersions', async () => { const findResult = await savedObjectsClient.find< diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index b9ee17f48360a..77cb16fcb8d5b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -83,6 +83,9 @@ export const importRules = async ({ const prebuiltRuleAssets = await prebuiltRulesImporter.fetchPrebuiltRuleAssets({ rules: batchParseObjects, }); + const installedRuleIds = await prebuiltRulesImporter.fetchInstalledRuleIds({ + rules: batchParseObjects, + }); const newImportRuleResponse = await Promise.all( batchParseObjects.reduce>>((accum, parsedRule) => { @@ -140,7 +143,7 @@ export const importRules = async ({ const ruleSource = calculateRuleSourceForImport({ rule: parsedRule, prebuiltRuleAssets, - installedRuleIds: [], // TODO + installedRuleIds, }); parsedRule.rule_source = ruleSource; From 554f340d0000eb1bb6f319d681fac34d8d377b47 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 13:17:55 -0500 Subject: [PATCH 035/120] Remove source_updated_at from rule schema We need to coordinate this addition with the TRaDE team (https://github.com/elastic/detection-rules/issues/2826). --- .../model/rule_schema/common_attributes.gen.ts | 7 ------- .../model/rule_schema/common_attributes.schema.yaml | 8 -------- ...olution_detections_api_2023_10_31.bundled.schema.yaml | 9 --------- ...olution_detections_api_2023_10_31.bundled.schema.yaml | 9 --------- .../model/rule_assets/prebuilt_rule_asset.ts | 2 -- .../rule_source/calculate_rule_source_from_asset.test.ts | 2 -- .../rule_source/calculate_rule_source_from_asset.ts | 1 - .../import/calculate_rule_source_for_import.test.ts | 2 -- 8 files changed, 40 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts index a3c81fd1c87b8..f5f1ca1306232 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts @@ -57,12 +57,6 @@ export const KqlQueryLanguageEnum = KqlQueryLanguage.enum; export type IsRuleImmutable = z.infer; export const IsRuleImmutable = z.boolean(); -/** - * The date and time that the external/prebuilt rule was last updated in its source repository. - */ -export type ExternalSourceUpdatedAt = z.infer; -export const ExternalSourceUpdatedAt = z.string().datetime(); - /** * Determines whether an external/prebuilt rule has been customized by the user (i.e. any of its fields have been modified and diverged from the base value). */ @@ -84,7 +78,6 @@ export type ExternalRuleSource = z.infer; export const ExternalRuleSource = z.object({ type: z.literal('external'), is_customized: IsExternalRuleCustomized, - source_updated_at: ExternalSourceUpdatedAt.optional(), }); /** diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml index aa56be14120ec..9864ba4b160b3 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml @@ -45,12 +45,6 @@ components: deprecated: true description: '[DEPRECATION WARNING TODO] - This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field.' - ExternalSourceUpdatedAt: - type: string - format: date-time - example: '2021-08-31T00:00:00Z' - description: The date and time that the external/prebuilt rule was last updated in its source repository. - IsExternalRuleCustomized: type: boolean description: Determines whether an external/prebuilt rule has been customized by the user (i.e. any of its fields have been modified and diverged from the base value). @@ -76,8 +70,6 @@ components: - external is_customized: $ref: '#/components/schemas/IsExternalRuleCustomized' - source_updated_at: - $ref: '#/components/schemas/ExternalSourceUpdatedAt' required: - type - is_customized diff --git a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml index 617e5e167c11f..9c404464889d8 100644 --- a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -2968,8 +2968,6 @@ components: properties: is_customized: $ref: '#/components/schemas/IsExternalRuleCustomized' - source_updated_at: - $ref: '#/components/schemas/ExternalSourceUpdatedAt' type: enum: - external @@ -2977,13 +2975,6 @@ components: required: - type - is_customized - ExternalSourceUpdatedAt: - description: >- - The date and time that the external/prebuilt rule was last updated in - its source repository. - example: '2021-08-31T00:00:00Z' - format: date-time - type: string FindRulesSortField: enum: - created_at diff --git a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml index 5c99a0676e615..587e33bfe708a 100644 --- a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -2242,8 +2242,6 @@ components: properties: is_customized: $ref: '#/components/schemas/IsExternalRuleCustomized' - source_updated_at: - $ref: '#/components/schemas/ExternalSourceUpdatedAt' type: enum: - external @@ -2251,13 +2249,6 @@ components: required: - type - is_customized - ExternalSourceUpdatedAt: - description: >- - The date and time that the external/prebuilt rule was last updated in - its source repository. - example: '2021-08-31T00:00:00Z' - format: date-time - type: string FindRulesSortField: enum: - created_at diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts index d154b4a440a6f..fa6c78bb7a8c1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/model/rule_assets/prebuilt_rule_asset.ts @@ -72,7 +72,6 @@ function zodMaskFor() { * Big differences between this schema and RuleCreateProps: * - rule_id is a required field * - version is a required field - * - source_updated_at is a new, optional field in support of prebuilt rule customization * - some fields are omitted because they are not present in https://github.com/elastic/detection-rules */ export type PrebuiltRuleAsset = z.infer; @@ -82,6 +81,5 @@ export const PrebuiltRuleAsset = BaseCreateProps.omit(BASE_PROPS_REMOVED_FROM_PR z.object({ rule_id: RuleSignatureId, version: RuleVersion, - source_updated_at: z.string().datetime().optional(), }) ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts index 31a096b295561..02871be57bd1c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts @@ -21,7 +21,6 @@ const buildTestRuleAsset = (overrides?: Partial) => { return { rule_id: 'rule_id', version: 1, - source_updated_at: '2024-05-01', ...overrides, } as PrebuiltRuleAsset; }; @@ -76,7 +75,6 @@ describe('calculateRuleSourceFromAsset', () => { expect(result).toEqual({ type: 'external', - source_updated_at: '2024-05-01', is_customized: true, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts index 5da267d8b289b..c60a694847583 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts @@ -40,6 +40,5 @@ export const calculateRuleSourceFromAsset = ({ return { type: 'external', is_customized: isCustomized, - source_updated_at: prebuiltRuleAsset.source_updated_at, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts index 8200834c98921..e96908d21206c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -21,7 +21,6 @@ const buildTestRuleAsset = (overrides?: Partial) => { return { rule_id: 'rule_id', version: 1, - source_updated_at: '2024-05-01', ...overrides, } as PrebuiltRuleAsset; }; @@ -64,7 +63,6 @@ describe('calculateRuleSourceForImport', () => { expect(result).toEqual({ type: 'external', - source_updated_at: '2024-05-01', isCustomized: true, }); }); From 49a166d6499627252a751abfa97e2355c1999497 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 13:25:17 -0500 Subject: [PATCH 036/120] Remove deprecation placeholder from immutable field We're not able to deprecate this field at this time, but we can instead base our logic on our new fields. --- .../model/rule_schema/common_attributes.gen.ts | 2 +- .../model/rule_schema/common_attributes.schema.yaml | 2 +- ...ity_solution_detections_api_2023_10_31.bundled.schema.yaml | 4 ++-- ...ity_solution_detections_api_2023_10_31.bundled.schema.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts index f5f1ca1306232..1bf3bfb5e2445 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts @@ -51,7 +51,7 @@ export type KqlQueryLanguageEnum = typeof KqlQueryLanguage.enum; export const KqlQueryLanguageEnum = KqlQueryLanguage.enum; /** - * [DEPRECATION WARNING TODO] - This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field. + * This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field. * @deprecated */ export type IsRuleImmutable = z.infer; diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml index 9864ba4b160b3..f1615a15fb3dc 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml @@ -43,7 +43,7 @@ components: IsRuleImmutable: type: boolean deprecated: true - description: '[DEPRECATION WARNING TODO] - This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field.' + description: 'This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field.' IsExternalRuleCustomized: type: boolean diff --git a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml index 9c404464889d8..b854dc2bc4475 100644 --- a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -3086,8 +3086,8 @@ components: IsRuleImmutable: deprecated: true description: >- - [DEPRECATION WARNING TODO] - This field determines whether the rule is a - prebuilt Elastic rule. It will be replaced with the `rule_source` field. + This field determines whether the rule is a prebuilt Elastic rule. It + will be replaced with the `rule_source` field. type: boolean ItemsPerSearch: minimum: 1 diff --git a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml index 587e33bfe708a..3e182c1112d5d 100644 --- a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -2337,8 +2337,8 @@ components: IsRuleImmutable: deprecated: true description: >- - [DEPRECATION WARNING TODO] - This field determines whether the rule is a - prebuilt Elastic rule. It will be replaced with the `rule_source` field. + This field determines whether the rule is a prebuilt Elastic rule. It + will be replaced with the `rule_source` field. type: boolean ItemsPerSearch: minimum: 1 From 31fc00c955a343659e703aad551ab0a51da7edee Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 13:32:32 -0500 Subject: [PATCH 037/120] Remove `immutable` from our incoming rule type We don't use this field for import now, so we can drop it here and just ignore it. --- .../rule_management/import_rules/rule_to_import.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index 8fbe461ff6747..958c7561fc1ae 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -30,7 +30,6 @@ export type RuleToImportInput = z.input; export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( ResponseFields.partial().extend({ rule_id: RuleSignatureId, - immutable: z.boolean().optional().default(false), /* Overriding `required_fields` from ResponseFields because in ResponseFields `required_fields` has the output type, From 75b0e3e4011ee418d4cad19d465a1eafd377535f Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 15:03:13 -0500 Subject: [PATCH 038/120] Remove requirement on immutable for rule creation This attribute can be regenerated from the other rule properties, and so its value (which is now defaulted) doesn't really matter here. --- .../logic/detection_rules_client/methods/create_rule.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/create_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/create_rule.ts index 7bd3c9c46bf73..c4ffbdef203ee 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/create_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/create_rule.ts @@ -24,7 +24,7 @@ interface CreateRuleOptions { actionsClient: ActionsClient; rulesClient: RulesClient; mlAuthz: MlAuthz; - rule: RuleCreateProps & { immutable: boolean }; + rule: RuleCreateProps; id?: string; allowMissingConnectorSecrets?: boolean; } From dba912b7bdfc959d876a448d7ba9ea36edcf2790 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 15:00:23 -0500 Subject: [PATCH 039/120] Version is required in rule import * Unifies PrebuiltRuleToImport with existing RuleToImport * Removes unnecessary logic now covered by zod validation * Updates (some) mocks and tests with new version requirement --- .../import_rules/rule_to_import.mock.ts | 3 ++ .../import_rules/rule_to_import.test.ts | 51 ++++++++++++++----- .../import_rules/rule_to_import.ts | 11 +--- .../normalization/convert_rule_to_diffable.ts | 6 +-- .../rule_source/calculate_is_customized.ts | 4 +- .../calculate_rule_source_from_asset.ts | 4 +- .../calculate_rule_source_for_import.ts | 4 +- .../logic/import/import_rules_utils.test.ts | 1 + .../logic/import/import_rules_utils.ts | 25 +-------- 9 files changed, 55 insertions(+), 54 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts index 2b36645363edc..8d1a2bd8f859d 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts @@ -18,6 +18,7 @@ export const getImportRulesSchemaMock = (rewrites?: Partial): Rule language: 'kuery', rule_id: 'rule-1', immutable: false, + version: 1, ...rewrites, } as RuleToImport); @@ -31,6 +32,7 @@ export const getImportRulesWithIdSchemaMock = (ruleId = 'rule-1'): RuleToImport risk_score: 55, language: 'kuery', rule_id: ruleId, + version: 1, immutable: false, }); @@ -87,6 +89,7 @@ export const getImportThreatMatchRulesSchemaMock = ( }, ], immutable: false, + version: 1, ...rewrites, } as RuleToImport); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts index 8e7415b7c2729..537c573c29e8b 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts @@ -22,7 +22,7 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 1 more"` + `"name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 2 more"` ); }); @@ -47,7 +47,7 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql'"` + `"name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 1 more"` ); }); @@ -61,7 +61,7 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"name: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql'"` + `"name: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', version: Required"` ); }); @@ -76,7 +76,7 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"name: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql'"` + `"name: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', version: Required"` ); }); @@ -90,6 +90,7 @@ describe('RuleToImport', () => { severity: 'low', interval: '5m', type: 'query', + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -98,6 +99,17 @@ describe('RuleToImport', () => { expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"risk_score: Required"`); }); + test('missing version is not valid', () => { + const payload: RuleToImportInput = getImportRulesSchemaMock(); + // @ts-expect-error version is normally required + delete payload.version; + + const result = RuleToImport.safeParse(payload); + expectParseError(result); + + expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"version: Required"`); + }); + test('[rule_id, description, from, to, name, severity, type, interval, index] does not validate', () => { const payload: Partial = { rule_id: 'rule-1', @@ -109,6 +121,7 @@ describe('RuleToImport', () => { type: 'query', interval: '5m', index: ['index-1'], + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -117,7 +130,7 @@ describe('RuleToImport', () => { expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"risk_score: Required"`); }); - test('[rule_id, description, from, to, name, severity, type, query, index, interval] does validate', () => { + test('[rule_id, description, from, to, name, severity, type, query, index, interval, version] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', risk_score: 50, @@ -130,6 +143,7 @@ describe('RuleToImport', () => { query: 'some query', index: ['index-1'], interval: '5m', + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -137,7 +151,7 @@ describe('RuleToImport', () => { expectParseSuccess(result); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, query, language] does not validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, version] does not validate', () => { const payload: Partial = { rule_id: 'rule-1', description: 'some description', @@ -150,6 +164,7 @@ describe('RuleToImport', () => { type: 'query', query: 'some query', language: 'kuery', + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -158,7 +173,7 @@ describe('RuleToImport', () => { expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"risk_score: Required"`); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score] does validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score, version] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', risk_score: 50, @@ -172,6 +187,7 @@ describe('RuleToImport', () => { type: 'query', query: 'some query', language: 'kuery', + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -179,7 +195,7 @@ describe('RuleToImport', () => { expectParseSuccess(result); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score, output_index] does validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score, output_index, version] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', output_index: '.siem-signals', @@ -194,6 +210,7 @@ describe('RuleToImport', () => { type: 'query', query: 'some query', language: 'kuery', + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -213,6 +230,7 @@ describe('RuleToImport', () => { interval: '5m', type: 'query', risk_score: 50, + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -233,6 +251,7 @@ describe('RuleToImport', () => { severity: 'low', interval: '5m', type: 'query', + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -261,6 +280,7 @@ describe('RuleToImport', () => { severity: 'low', interval: '5m', type: 'query', + version: 1, threat: [ { framework: 'someFramework', @@ -880,7 +900,7 @@ describe('RuleToImport', () => { ); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note] does validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and version] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', description: 'some description', @@ -893,6 +913,7 @@ describe('RuleToImport', () => { type: 'query', risk_score: 50, note: '# some markdown', + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -901,7 +922,7 @@ describe('RuleToImport', () => { }); describe('exception_list', () => { - test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and exceptions_list] does validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, version, and exceptions_list] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', description: 'some description', @@ -916,13 +937,14 @@ describe('RuleToImport', () => { risk_score: 50, note: '# some markdown', exceptions_list: getListArrayMock(), + version: 1, }; const result = RuleToImport.safeParse(payload); expectParseSuccess(result); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, version, and empty exceptions_list] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', description: 'some description', @@ -937,6 +959,7 @@ describe('RuleToImport', () => { risk_score: 50, note: '# some markdown', exceptions_list: [], + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -957,6 +980,7 @@ describe('RuleToImport', () => { filters: [], risk_score: 50, note: '# some markdown', + version: 1, exceptions_list: [{ id: 'uuid_here', namespace_type: 'not a namespace type' }], }; @@ -968,7 +992,7 @@ describe('RuleToImport', () => { ); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, version, and non-existent exceptions_list] does validate with empty exceptions_list', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', description: 'some description', @@ -982,6 +1006,7 @@ describe('RuleToImport', () => { filters: [], risk_score: 50, note: '# some markdown', + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -1012,6 +1037,7 @@ describe('RuleToImport', () => { data_view_id: 'logs-*', index: [], interval: '5m', + version: 1, }; const result = RuleToImport.safeParse(payload); @@ -1033,6 +1059,7 @@ describe('RuleToImport', () => { data_view_id: 'logs-*', index: ['auditbeat-*'], interval: '5m', + version: 1, }; const result = RuleToImport.safeParse(payload); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index 958c7561fc1ae..66b18aee6c2d6 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -18,6 +18,7 @@ import { /** * Differences from this and the createRulesSchema are * - rule_id is required + * - version is required * - id is optional (but ignored in the import code - rule_id is exclusively used for imports) * - immutable is optional (but ignored in the import code) * - created_at is optional (but ignored in the import code) @@ -30,6 +31,7 @@ export type RuleToImportInput = z.input; export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( ResponseFields.partial().extend({ rule_id: RuleSignatureId, + version: RuleVersion, /* Overriding `required_fields` from ResponseFields because in ResponseFields `required_fields` has the output type, @@ -40,12 +42,3 @@ export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( required_fields: z.array(RequiredFieldInput).optional(), }) ); - -/** - * PrebuiltRuleToImport represents a RuleToImport that has been validated as - * representing a prebuilt rule. Part of this definition is the inclusion of a - * "version" specifier, which is represented here. - */ -export type PrebuiltRuleToImport = z.infer; -export type PrebuiltRuleToImportInput = z.input; -export const PrebuiltRuleToImport = RuleToImport.and(z.object({ version: RuleVersion })); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts index 6467ec7911f2b..1b19d3d5d57d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/normalization/convert_rule_to_diffable.ts @@ -9,7 +9,7 @@ import type { RequiredOptional } from '@kbn/zod-helpers'; import { requiredOptional } from '@kbn/zod-helpers'; import { DEFAULT_MAX_SIGNALS } from '../../../../../../../common/constants'; import { assertUnreachable } from '../../../../../../../common/utility_types'; -import type { PrebuiltRuleToImport } from '../../../../../../../common/api/detection_engine'; +import type { RuleToImport } from '../../../../../../../common/api/detection_engine'; import type { EqlRule, EqlRuleCreateProps, @@ -61,7 +61,7 @@ import { addEcsToRequiredFields } from '../../../../rule_management/utils/utils' * Read more in the JSDoc description of DiffableRule. */ export const convertRuleToDiffable = ( - rule: RuleResponse | PrebuiltRuleAsset | PrebuiltRuleToImport + rule: RuleResponse | PrebuiltRuleAsset | RuleToImport ): DiffableRule => { const commonFields = extractDiffableCommonFields(rule); @@ -112,7 +112,7 @@ export const convertRuleToDiffable = ( }; const extractDiffableCommonFields = ( - rule: RuleResponse | PrebuiltRuleAsset | PrebuiltRuleToImport + rule: RuleResponse | PrebuiltRuleAsset | RuleToImport ): RequiredOptional => { return { // --------------------- REQUIRED FIELDS diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts index 8ea85eda78e17..f9dfa6d3c0a57 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts @@ -6,7 +6,7 @@ */ import type { - PrebuiltRuleToImport, + RuleToImport, RuleResponse, } from '../../../../../../../../common/api/detection_engine'; import { MissingVersion } from '../../../../../../../../common/api/detection_engine'; @@ -17,7 +17,7 @@ import { convertPrebuiltRuleAssetToRuleResponse } from '../../converters/convert export function calculateIsCustomized( baseRule: PrebuiltRuleAsset | undefined, - nextRule: RuleResponse | PrebuiltRuleToImport + nextRule: RuleResponse | RuleToImport ) { if (baseRule == null) { // If the base version is missing, we consider the rule to be customized diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts index c60a694847583..98d93382e6f66 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts @@ -8,7 +8,7 @@ import type { RuleSource, RuleResponse, - PrebuiltRuleToImport, + RuleToImport, } from '../../../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateIsCustomized } from './calculate_is_customized'; @@ -18,7 +18,7 @@ export const calculateRuleSourceFromAsset = ({ prebuiltRuleAsset, ruleIdExists, }: { - rule: RuleResponse | PrebuiltRuleToImport; + rule: RuleResponse | RuleToImport; prebuiltRuleAsset: PrebuiltRuleAsset | undefined; ruleIdExists: boolean; }): RuleSource => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 5d7f5db991afa..9ae4d26add512 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -6,7 +6,7 @@ */ import type { - PrebuiltRuleToImport, + RuleToImport, RuleSource, } from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; @@ -25,7 +25,7 @@ export const calculateRuleSourceForImport = ({ prebuiltRuleAssets, installedRuleIds, }: { - rule: PrebuiltRuleToImport; + rule: RuleToImport; prebuiltRuleAssets: PrebuiltRuleAsset[]; installedRuleIds: string[]; }): RuleSource => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts index d78d899ebafdd..84bdb9c583a2d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts @@ -148,6 +148,7 @@ describe('importRules', () => { }); it('rejects a prebuilt rule when version is missing', async () => { + // @ts-expect-error version is required on the type delete prebuiltRuleToImport.version; const ruleChunk = [prebuiltRuleToImport]; const result = await importRules({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 77cb16fcb8d5b..889f077c7bf17 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -12,10 +12,7 @@ import type { ImportExceptionListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import type { - PrebuiltRuleToImport, - RuleToImport, -} from '../../../../../../common/api/detection_engine/rule_management'; +import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; import type { PrebuiltRulesImporter } from '../../../prebuilt_rules/logic/prebuilt_rules_importer'; @@ -122,24 +119,6 @@ export const importRules = async ({ } if (allowPrebuiltRules) { - // TODO how do we differentiate this case from a non-prebuilt import? - if (!ruleImportIsPrebuilt(parsedRule)) { - resolve( - createBulkErrorObject({ - statusCode: 400, - message: i18n.translate( - 'xpack.securitySolution.detectionEngine.rules.cannotImportPrebuiltRuleWithoutVersion', - { - defaultMessage: 'Prebuilt rules must specify a "version" to be imported.', - } - ), - ruleId: parsedRule.rule_id, - }) - ); - - return null; - } - const ruleSource = calculateRuleSourceForImport({ rule: parsedRule, prebuiltRuleAssets, @@ -192,5 +171,3 @@ export const importRules = async ({ return importRuleResponse; }; - -const ruleImportIsPrebuilt = (rule: RuleToImport): rule is PrebuiltRuleToImport => !!rule.version; From 91841c602a55b94ebeea2de76bf26a104e702ea8 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 15:21:32 -0500 Subject: [PATCH 040/120] Clarify method logic in comments --- .../prebuilt_rules/logic/prebuilt_rules_importer.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts index a4e08353ba87b..33a1e496d5502 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts @@ -60,11 +60,12 @@ export class PrebuiltRulesImporter implements IPrebuiltRulesImporter { } /** - * Retrieves the prebuilt rule assets for the specified rules being imported. If - * @param rules - The rules to be imported + * Retrieves prebuilt rule assets for rules that are being imported. + * @param rules - The rules being imported * * @returns The prebuilt rule assets corresponding to the specified prebuilt * rules, which are used to determine how to import those rules (create vs. update, etc.). + * Assets match the `rule_id` and `version` of the specified rules. */ public async fetchPrebuiltRuleAssets({ rules, @@ -95,7 +96,7 @@ export class PrebuiltRulesImporter implements IPrebuiltRulesImporter { } /** - * Retrieves the rule IDs (`rule_id`) of installed prebuilt rules matching those + * Retrieves the rule IDs (`rule_id`s) of installed prebuilt rules matching those * of the specified rules. Can be used to determine whether the rule being * imported is a custom rule or a prebuilt rule. * From 7fab1086a4d13c8e6bd931578ee79d1787211995 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 15:22:20 -0500 Subject: [PATCH 041/120] Add missing await --- .../rule_management/logic/import/import_rules_utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 889f077c7bf17..4e3656068b69b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -69,7 +69,7 @@ export const importRules = async ({ return importRuleResponse; } - prebuiltRulesImporter.setup(); + await prebuiltRulesImporter.setup(); while (ruleChunks.length) { const batchParseObjects = ruleChunks.shift() ?? []; From b47153225492d5fcaa5b703d14fb3e7248a03ee6 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 16:09:02 -0500 Subject: [PATCH 042/120] Update rule_source calculation * Updates logic so that is_customized: true when rule is found without a matching version * Uses existing test mocks to satisfy zod validation * Updates parameters and comments to be more explicit --- .../calculate_rule_source_from_asset.test.ts | 85 +++++++++---------- .../calculate_rule_source_from_asset.ts | 21 +++-- .../calculate_rule_source_for_import.test.ts | 38 +++------ .../calculate_rule_source_for_import.ts | 13 +-- 4 files changed, 76 insertions(+), 81 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts index 02871be57bd1c..570abab48c4c6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts @@ -5,31 +5,15 @@ * 2.0. */ -import type { RuleResponse } from '../../../../../../../../common/api/detection_engine'; -import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset'; - -const buildTestRule = (overrides?: Partial) => { - return { - rule_id: 'rule_id', - version: 1, - ...overrides, - } as RuleResponse; -}; - -const buildTestRuleAsset = (overrides?: Partial) => { - return { - rule_id: 'rule_id', - version: 1, - ...overrides, - } as PrebuiltRuleAsset; -}; +import { getRulesSchemaMock } from '../../../../../../../../common/api/detection_engine/model/rule_schema/mocks'; +import { getPrebuiltRuleMock } from '../../../../../prebuilt_rules/mocks'; describe('calculateRuleSourceFromAsset', () => { it('calculates as internal if no asset is found', () => { const result = calculateRuleSourceFromAsset({ - rule: buildTestRule(), - prebuiltRuleAsset: undefined, + rule: getRulesSchemaMock(), + assetWithMatchingVersion: undefined, ruleIdExists: false, }); @@ -38,44 +22,55 @@ describe('calculateRuleSourceFromAsset', () => { }); }); - it('calculates as unmodified external type if an asset is found without a matching version', () => { + it('calculates as customized external type if an asset is found matching rule_id but not version', () => { + const ruleToImport = getRulesSchemaMock(); const result = calculateRuleSourceFromAsset({ - rule: buildTestRule(), - prebuiltRuleAsset: buildTestRuleAsset({ version: 2 }), + rule: ruleToImport, + assetWithMatchingVersion: undefined, ruleIdExists: true, }); expect(result).toEqual({ type: 'external', - is_customized: false, + is_customized: true, }); }); - it('calculates as unmodified external type if an asset is not found but the rule ID exists', () => { - const result = calculateRuleSourceFromAsset({ - rule: buildTestRule(), - prebuiltRuleAsset: undefined, - ruleIdExists: true, - }); + describe('matching rule_id and version is found', () => { + it('calculates as customized external type if the imported rule has all fields unchanged from the asset', () => { + const ruleToImport = getRulesSchemaMock(); + const result = calculateRuleSourceFromAsset({ + rule: getRulesSchemaMock(), // version 1 + assetWithMatchingVersion: getPrebuiltRuleMock({ + ...ruleToImport, + version: 1, // version 1 (same version as imported rule) + // no other overwrites -> no differences + }), + ruleIdExists: true, + }); - expect(result).toEqual({ - type: 'external', - is_customized: false, + expect(result).toEqual({ + type: 'external', + is_customized: false, + }); }); - }); - - it('calculates as external with customizations if a matching asset/version is found', () => { - const rule = buildTestRule(); - const result = calculateRuleSourceFromAsset({ - rule, - prebuiltRuleAsset: buildTestRuleAsset(), - ruleIdExists: true, - }); + it('calculates as non-customized external type the imported rule has fields which differ from the asset', () => { + const ruleToImport = getRulesSchemaMock(); + const result = calculateRuleSourceFromAsset({ + rule: getRulesSchemaMock(), // version 1 + assetWithMatchingVersion: getPrebuiltRuleMock({ + ...ruleToImport, + version: 1, // version 1 (same version as imported rule) + name: 'Customized name', // mock a customization + }), + ruleIdExists: true, + }); - expect(result).toEqual({ - type: 'external', - is_customized: true, + expect(result).toEqual({ + type: 'external', + is_customized: true, + }); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts index 98d93382e6f66..568ce32254d8e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts @@ -13,13 +13,24 @@ import type { import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateIsCustomized } from './calculate_is_customized'; +/** + * Calculates rule_source for a rule based on two pieces of information: + * 1. The prebuilt rule asset that matches the specified rule_id and version + * 2. Whether a prebuilt rule with the specified rule_id is currently installed + * + * @param rule The rule for which rule_source is being calculated + * @param assetWithMatchingVersion The prebuilt rule asset that matches the specified rule_id and version + * @param ruleIdExists Whether a prebuilt rule with the specified rule_id is currently installed + * + * @returns The calculated rule_source + */ export const calculateRuleSourceFromAsset = ({ rule, - prebuiltRuleAsset, + assetWithMatchingVersion, ruleIdExists, }: { rule: RuleResponse | RuleToImport; - prebuiltRuleAsset: PrebuiltRuleAsset | undefined; + assetWithMatchingVersion: PrebuiltRuleAsset | undefined; ruleIdExists: boolean; }): RuleSource => { if (!ruleIdExists) { @@ -28,14 +39,14 @@ export const calculateRuleSourceFromAsset = ({ }; } - if (prebuiltRuleAsset == null) { + if (assetWithMatchingVersion == null) { return { type: 'external', - is_customized: false, + is_customized: true, }; } - const isCustomized = calculateIsCustomized(prebuiltRuleAsset, rule); + const isCustomized = calculateIsCustomized(assetWithMatchingVersion, rule); return { type: 'external', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts index e96908d21206c..38b3180462a46 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -5,30 +5,14 @@ * 2.0. */ -import type { RuleResponse } from '../../../../../../common/api/detection_engine'; -import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; +import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; +import { getPrebuiltRuleMock } from '../../../prebuilt_rules/mocks'; import { calculateRuleSourceForImport } from './calculate_rule_source_for_import'; -const buildTestRule = (overrides?: Partial) => { - return { - rule_id: 'rule_id', - version: 1, - ...overrides, - } as RuleResponse; -}; - -const buildTestRuleAsset = (overrides?: Partial) => { - return { - rule_id: 'rule_id', - version: 1, - ...overrides, - } as PrebuiltRuleAsset; -}; - describe('calculateRuleSourceForImport', () => { it('calculates as internal if no asset is found', () => { const result = calculateRuleSourceForImport({ - rule: buildTestRule(), + rule: getRulesSchemaMock(), prebuiltRuleAssets: [], installedRuleIds: [], }); @@ -38,22 +22,26 @@ describe('calculateRuleSourceForImport', () => { }); }); - it('calculates as unmodified external type if an asset is found without a matching version', () => { + it('calculates as modified external type if an asset is found without a matching version', () => { + const rule = getRulesSchemaMock(); + rule.rule_id = 'rule_id'; + const result = calculateRuleSourceForImport({ - rule: buildTestRule(), + rule, prebuiltRuleAssets: [], installedRuleIds: ['rule_id'], }); expect(result).toEqual({ type: 'external', - isCustomized: false, + is_customized: true, }); }); it('calculates as external with customizations if a matching asset/version is found', () => { - const rule = buildTestRule(); - const prebuiltRuleAssets = [buildTestRuleAsset()]; + const rule = getRulesSchemaMock(); + rule.rule_id = 'rule_id'; + const prebuiltRuleAssets = [getPrebuiltRuleMock({ rule_id: 'rule_id' })]; const result = calculateRuleSourceForImport({ rule, @@ -63,7 +51,7 @@ describe('calculateRuleSourceForImport', () => { expect(result).toEqual({ type: 'external', - isCustomized: true, + is_customized: true, }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 9ae4d26add512..f4f94ce13358d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -5,10 +5,7 @@ * 2.0. */ -import type { - RuleToImport, - RuleSource, -} from '../../../../../../common/api/detection_engine'; +import type { RuleToImport, RuleSource } from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; import { calculateRuleSourceFromAsset } from '../detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset'; @@ -19,6 +16,8 @@ import { calculateRuleSourceFromAsset } from '../detection_rules_client/mergers/ * @param prebuiltRuleAssets A list of prebuilt rule assets, which may include * the installed version of the specified prebuilt rule. * @param installedRuleIds A list of prebuilt rule IDs that are currently installed + * + * @returns The calculated rule_source */ export const calculateRuleSourceForImport = ({ rule, @@ -29,12 +28,14 @@ export const calculateRuleSourceForImport = ({ prebuiltRuleAssets: PrebuiltRuleAsset[]; installedRuleIds: string[]; }): RuleSource => { - const matchingAsset = prebuiltRuleAssets.find((asset) => asset.rule_id === rule.rule_id); + const assetWithMatchingVersion = prebuiltRuleAssets.find( + (asset) => asset.rule_id === rule.rule_id + ); const ruleIdExists = installedRuleIds.includes(rule.rule_id); return calculateRuleSourceFromAsset({ rule, - prebuiltRuleAsset: matchingAsset, + assetWithMatchingVersion, ruleIdExists, }); }; From 7a839ff528350665ec63bfad84adaabe81db91aa Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 16:37:34 -0500 Subject: [PATCH 043/120] Update import error message to include rule_id In the course of making this change I noticed that related unit tests were failing, since we were no longer explicitly rejecting a missing version. I deleted the test in import_rules_utils and moved it to the route, instead. --- .../api/rules/import_rules/route.test.ts | 36 +++++++++++++++++++ .../logic/import/import_rules_utils.test.ts | 31 ++-------------- .../logic/import/import_rules_utils.ts | 3 +- 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts index c9bc11384afa1..e0cf057f0d39a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts @@ -272,6 +272,42 @@ describe('Import rules route', () => { }); }); }); + + it('returns an error if rule is missing a version', async () => { + const ruleWithoutVersion = getImportRulesWithIdSchemaMock('rule-1'); + // @ts-expect-error + delete ruleWithoutVersion.version; + const payload = buildHapiStream(rulesToNdJsonString([ruleWithoutVersion])); + const versionlessRequest = getImportRulesRequest(payload); + + const response = await server.inject( + versionlessRequest, + requestContextMock.convertContext(context) + ); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ + errors: [ + { + error: { + message: `version: Required`, + status_code: 400, + }, + rule_id: '(unknown id)', + }, + ], + success: false, + success_count: 0, + rules_count: 1, + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], + }); + }); }); describe('multi rule import', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts index 84bdb9c583a2d..d039c8b715516 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts @@ -27,6 +27,8 @@ describe('importRules', () => { savedObjectsClient = savedObjectsClientMock.create(); mockPrebuiltRulesImporter = prebuiltRulesImporterMock.create(); + mockPrebuiltRulesImporter.fetchPrebuiltRuleAssets.mockResolvedValue([]); + mockPrebuiltRulesImporter.fetchInstalledRuleIds.mockResolvedValue([]); }); it('returns an empty rules response if no rules to import', async () => { @@ -138,34 +140,7 @@ describe('importRules', () => { expect(result).toEqual([ { error: { - message: - 'Importing prebuilt rules is not supported. To import this rule as a custom rule, remove its "immutable" property try again.', - status_code: 400, - }, - rule_id: prebuiltRuleToImport.rule_id, - }, - ]); - }); - - it('rejects a prebuilt rule when version is missing', async () => { - // @ts-expect-error version is required on the type - delete prebuiltRuleToImport.version; - const ruleChunk = [prebuiltRuleToImport]; - const result = await importRules({ - ruleChunks: [ruleChunk], - rulesResponseAcc: [], - overwriteRules: false, - detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImporter: mockPrebuiltRulesImporter, - allowPrebuiltRules: false, - savedObjectsClient, - }); - - expect(result).toEqual([ - { - error: { - message: - 'Importing prebuilt rules is not supported. To import this rule as a custom rule, remove its "immutable" property try again.', + message: `Importing prebuilt rules is not supported. To import this rule as a custom rule, first duplicate the rule and then export it. [rule_id: ${prebuiltRuleToImport.rule_id}]`, status_code: 400, }, rule_id: prebuiltRuleToImport.rule_id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 4e3656068b69b..4cc20c11e4e5d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -108,7 +108,8 @@ export const importRules = async ({ 'xpack.securitySolution.detectionEngine.rules.importPrebuiltRulesUnsupported', { defaultMessage: - 'Importing prebuilt rules is not supported. To import this rule as a custom rule, remove its "immutable" property try again.', + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, first duplicate the rule and then export it. [rule_id: {ruleId}]', + values: { ruleId: parsedRule.rule_id }, } ), ruleId: parsedRule.rule_id, From 74cc5e8b2d8cb86e198487c504c9b20db1c52cdf Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 17:38:04 -0500 Subject: [PATCH 044/120] Calculate `immutable` field for importing rules While we could theoretically calculate this field correctly later, we should calculate the proper value on import so that it's persisted correctly. --- .../calculate_rule_source_for_import.test.ts | 19 ++++++++++++++----- .../calculate_rule_source_for_import.ts | 11 ++++++++--- .../logic/import/import_rules_utils.ts | 3 ++- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts index 38b3180462a46..b35ddb4142429 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -18,7 +18,10 @@ describe('calculateRuleSourceForImport', () => { }); expect(result).toEqual({ - type: 'internal', + ruleSource: { + type: 'internal', + }, + immutable: false, }); }); @@ -33,8 +36,11 @@ describe('calculateRuleSourceForImport', () => { }); expect(result).toEqual({ - type: 'external', - is_customized: true, + ruleSource: { + type: 'external', + is_customized: true, + }, + immutable: true, }); }); @@ -50,8 +56,11 @@ describe('calculateRuleSourceForImport', () => { }); expect(result).toEqual({ - type: 'external', - is_customized: true, + ruleSource: { + type: 'external', + is_customized: true, + }, + immutable: true, }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index f4f94ce13358d..67d0e4f0231d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -17,7 +17,7 @@ import { calculateRuleSourceFromAsset } from '../detection_rules_client/mergers/ * the installed version of the specified prebuilt rule. * @param installedRuleIds A list of prebuilt rule IDs that are currently installed * - * @returns The calculated rule_source + * @returns The calculated rule_source and immutable fields for the rule */ export const calculateRuleSourceForImport = ({ rule, @@ -27,15 +27,20 @@ export const calculateRuleSourceForImport = ({ rule: RuleToImport; prebuiltRuleAssets: PrebuiltRuleAsset[]; installedRuleIds: string[]; -}): RuleSource => { +}): { ruleSource: RuleSource; immutable: boolean } => { const assetWithMatchingVersion = prebuiltRuleAssets.find( (asset) => asset.rule_id === rule.rule_id ); const ruleIdExists = installedRuleIds.includes(rule.rule_id); - return calculateRuleSourceFromAsset({ + const ruleSource = calculateRuleSourceFromAsset({ rule, assetWithMatchingVersion, ruleIdExists, }); + + return { + ruleSource, + immutable: ruleSource.type === 'external', + }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 4cc20c11e4e5d..e8a1effeed2b2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -120,13 +120,14 @@ export const importRules = async ({ } if (allowPrebuiltRules) { - const ruleSource = calculateRuleSourceForImport({ + const { immutable, ruleSource } = calculateRuleSourceForImport({ rule: parsedRule, prebuiltRuleAssets, installedRuleIds, }); parsedRule.rule_source = ruleSource; + parsedRule.immutable = immutable; } try { From 62554d9a8569b675a171fb9f888a21b8d584abbc Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 5 Sep 2024 18:31:38 -0500 Subject: [PATCH 045/120] Simplify interface of rule_source calculation We're only calling this on rule import, so no need to expect a ruleResponse as input. --- .../rule_source/calculate_rule_source_from_asset.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts index 568ce32254d8e..285b476067c0d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts @@ -5,11 +5,7 @@ * 2.0. */ -import type { - RuleSource, - RuleResponse, - RuleToImport, -} from '../../../../../../../../common/api/detection_engine'; +import type { RuleSource, RuleToImport } from '../../../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateIsCustomized } from './calculate_is_customized'; @@ -29,7 +25,7 @@ export const calculateRuleSourceFromAsset = ({ assetWithMatchingVersion, ruleIdExists, }: { - rule: RuleResponse | RuleToImport; + rule: RuleToImport; assetWithMatchingVersion: PrebuiltRuleAsset | undefined; ruleIdExists: boolean; }): RuleSource => { From c48b20ec1409d3948314e924efa214f55564533b Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 9 Sep 2024 20:52:47 -0500 Subject: [PATCH 046/120] Revert simplification of createRule function We're not even going to try to use this existing method, but instead write our own simplified one. --- .../logic/detection_rules_client/methods/create_rule.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/create_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/create_rule.ts index c4ffbdef203ee..7bd3c9c46bf73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/create_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/create_rule.ts @@ -24,7 +24,7 @@ interface CreateRuleOptions { actionsClient: ActionsClient; rulesClient: RulesClient; mlAuthz: MlAuthz; - rule: RuleCreateProps; + rule: RuleCreateProps & { immutable: boolean }; id?: string; allowMissingConnectorSecrets?: boolean; } From 99269648693dd8c182b928b27d2a9b6411fd2c8e Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 9 Sep 2024 20:54:07 -0500 Subject: [PATCH 047/120] Add note from investigation On import we call this function when the rule already exists; however, when we transform that rule to an "alert" we drop the value entirely. This value might matter in another call, or it might not: needs further investigation --- .../logic/detection_rules_client/mergers/apply_rule_update.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts index b911e66a1fc45..244431c0d9d48 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts @@ -28,7 +28,7 @@ export const applyRuleUpdate = async ({ ...applyRuleDefaults(ruleUpdate), // Use existing values - enabled: ruleUpdate.enabled ?? existingRule.enabled, + enabled: ruleUpdate.enabled ?? existingRule.enabled, // TODO this value isn't used by our call to update version: ruleUpdate.version ?? existingRule.version, // Always keep existing values for these fields From 1853d745e148714173b6d2729a1b09433132bb71 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 9 Sep 2024 20:56:01 -0500 Subject: [PATCH 048/120] Add new function for importing rules, both prebuilt and custom Now that we calculate proper values for `rule_source` and `immutable` in the outer loop of `importRules`, the simplified, temporary calculations that were previously happening in `importRule` are no longer necessary. Rather than extending the existing function to support both situations, I've taken a stab at writing a new function. It currently looks much the same as the existing one, but I'm now going to start paring it down. --- .../methods/import_rule_with_source.test.ts | 325 ++++++++++++++++++ .../methods/import_rule_with_source.ts | 105 ++++++ 2 files changed, 430 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts new file mode 100644 index 0000000000000..3f47d7a27f6d9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts @@ -0,0 +1,325 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { actionsClientMock } from '@kbn/actions-plugin/server/actions_client/actions_client.mock'; +import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; + +import { getImportRulesSchemaMock } from '../../../../../../../common/api/detection_engine/rule_management/mocks'; +import { importRuleWithSource } from './import_rule_with_source'; +import { buildMlAuthz } from '../../../../../machine_learning/authz'; +import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; +import { + getEmptyFindResult, + getFindResultWithSingleHit, + getRuleMock, +} from '../../../../routes/__mocks__/request_responses'; +import { getQueryRuleParams } from '../../../../rule_schema/mocks'; + +jest.mock('../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'); +jest.mock('../../../../../machine_learning/authz'); + +describe('importRuleWithSource', () => { + let mockActionsClient: ReturnType; + let mockRulesClient: ReturnType; + let mockMlAuthz: ReturnType; + let mockPrebuiltRuleAssetsClient: ReturnType; + + let params: Omit[0], 'importRulePayload'>; + + beforeEach(() => { + mockActionsClient = actionsClientMock.create(); + mockRulesClient = rulesClientMock.create(); + mockRulesClient.create.mockResolvedValue(getRuleMock(getQueryRuleParams())); + mockRulesClient.update.mockResolvedValue(getRuleMock(getQueryRuleParams())); + mockMlAuthz = (buildMlAuthz as jest.Mock)(); + mockPrebuiltRuleAssetsClient = (createPrebuiltRuleAssetsClient as jest.Mock)(); + params = { + actionsClient: mockActionsClient, + rulesClient: mockRulesClient, + prebuiltRuleAssetClient: mockPrebuiltRuleAssetsClient, + mlAuthz: mockMlAuthz, + }; + }); + + describe('when importing an existing rule', () => { + let existingRule: ReturnType['data'][0]; + + beforeEach(() => { + mockRulesClient.find.mockResolvedValue(getFindResultWithSingleHit()); + [existingRule] = getFindResultWithSingleHit().data; + }); + + it('throws an error if overwriting is disabled', async () => { + const rule = { + ...getImportRulesSchemaMock(), + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await expect( + importRuleWithSource({ + ...params, + importRulePayload: { + ruleToImport: rule, + overwriteRules: false, + }, + }) + ).rejects.toMatchObject({ + error: { + message: 'rule_id: "rule-1" already exists', + status_code: 409, + }, + }); + }); + + it('preserves the passed "rule_source" and "immutable" values', async () => { + const rule = { + ...getImportRulesSchemaMock(), + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await importRuleWithSource({ + ...params, + importRulePayload: { + ruleToImport: rule, + overwriteRules: true, + }, + }); + + expect(mockRulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + immutable: true, + ruleSource: { + isCustomized: true, + type: 'external', + }, + }), + }), + }) + ); + }); + + it('TODO REVIEW does not preserve the passed "enabled" value', async () => { + const disabledRule = { + ...getImportRulesSchemaMock(), + enabled: false, + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await importRuleWithSource({ + ...params, + importRulePayload: { + ruleToImport: disabledRule, + overwriteRules: true, + }, + }); + + expect(mockRulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.not.objectContaining({ + enabled: expect.anything(), + }), + }) + ); + }); + + it('TODO REVIEW preserves the existing rule\'s "id" value', async () => { + const rule = { + ...getImportRulesSchemaMock(), + immutable: true, + id: 'some-id', + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await importRuleWithSource({ + ...params, + importRulePayload: { + ruleToImport: rule, + overwriteRules: true, + }, + }); + + expect(mockRulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + id: existingRule.id, + data: expect.objectContaining({ + params: expect.objectContaining({ + version: existingRule.params.version, + }), + }), + }) + ); + }); + + it('TODO REVIEW preserves the existing rule\'s "version" value if left unspecified', async () => { + const rule = { + ...getImportRulesSchemaMock(), + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await importRuleWithSource({ + ...params, + importRulePayload: { + ruleToImport: rule, + overwriteRules: true, + }, + }); + + expect(mockRulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + version: existingRule.params.version, + }), + }), + }) + ); + }); + + it('TODO REVIEW preserves the specified "version" value', async () => { + const rule = { + ...getImportRulesSchemaMock(), + immutable: true, + version: 42, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await importRuleWithSource({ + ...params, + importRulePayload: { + ruleToImport: rule, + overwriteRules: true, + }, + }); + + expect(mockRulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + version: 42, + }), + }), + }) + ); + }); + }); + + describe('when importing a new rule', () => { + beforeEach(() => { + mockRulesClient.find.mockResolvedValueOnce(getEmptyFindResult()); + }); + + it('preserves the passed "rule_source" and "immutable" values', async () => { + const rule = { + ...getImportRulesSchemaMock(), + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await importRuleWithSource({ + ...params, + importRulePayload: { + ruleToImport: rule, + }, + }); + + expect(mockRulesClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + immutable: true, + ruleSource: { + isCustomized: true, + type: 'external', + }, + }), + }), + }) + ); + }); + + it('preserves the passed "enabled" value', async () => { + const rule = { + ...getImportRulesSchemaMock(), + enabled: true, + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await importRuleWithSource({ + ...params, + importRulePayload: { + ruleToImport: rule, + overwriteRules: true, + }, + }); + + expect(mockRulesClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + enabled: true + }), + }) + ); + }); + + it('defaults defaultable values', async () => { + const rule = { + ...getImportRulesSchemaMock(), + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await importRuleWithSource({ + ...params, + importRulePayload: { + ruleToImport: rule, + }, + }); + + expect(mockRulesClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + actions: [], + }), + }) + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts new file mode 100644 index 0000000000000..29f64e7cc2189 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import { ruleTypeMappings } from '@kbn/securitysolution-rules'; + +import type { + RuleResponse, + RuleSource, +} from '../../../../../../../common/api/detection_engine/model/rule_schema'; +import type { MlAuthz } from '../../../../../machine_learning/authz'; +import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; +import { createBulkErrorObject } from '../../../../routes/utils'; +import { convertAlertingRuleToRuleResponse } from '../converters/convert_alerting_rule_to_rule_response'; +import { convertRuleResponseToAlertingRule } from '../converters/convert_rule_response_to_alerting_rule'; +import type { ImportRuleArgs } from '../detection_rules_client_interface'; +import { applyRuleUpdate } from '../mergers/apply_rule_update'; +import { applyRuleDefaults } from '../mergers/apply_rule_defaults'; +import { validateMlAuth } from '../utils'; +import { getRuleByRuleId } from './get_rule_by_rule_id'; +import { SERVER_APP_ID } from '../../../../../../../common'; + +interface ImportRuleWithSourceArgs extends ImportRuleArgs { + ruleToImport: ImportRuleArgs['ruleToImport'] & { rule_source: RuleSource; immutable: boolean }; +} + +interface ImportRuleOptions { + actionsClient: ActionsClient; + rulesClient: RulesClient; + prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; + importRulePayload: ImportRuleWithSourceArgs; + mlAuthz: MlAuthz; +} + +/** + * Imports a rule with pre-calculated `rule_source` (and `immutable`) fields. + * Once prebuilt rule customization epic is complete, this function can be used + * to import all rules. Until then, this function should only be used when the feature flag is enabled. + * + */ +export const importRuleWithSource = async ({ + actionsClient, + rulesClient, + importRulePayload, + prebuiltRuleAssetClient, + mlAuthz, +}: ImportRuleOptions): Promise => { + const { ruleToImport, overwriteRules, allowMissingConnectorSecrets } = importRulePayload; + const { rule_source: ruleSource, immutable } = ruleToImport; + + await validateMlAuth(mlAuthz, ruleToImport.type); + + const existingRule = await getRuleByRuleId({ + rulesClient, + ruleId: ruleToImport.rule_id, + }); + + if (existingRule && !overwriteRules) { + throw createBulkErrorObject({ + ruleId: existingRule.rule_id, + statusCode: 409, + message: `rule_id: "${existingRule.rule_id}" already exists`, + }); + } + + if (existingRule && overwriteRules) { + const ruleWithUpdates = await applyRuleUpdate({ + prebuiltRuleAssetClient, + existingRule, + ruleUpdate: ruleToImport, + }); + ruleWithUpdates.rule_source = ruleSource; + ruleWithUpdates.immutable = immutable; + + const updatedRule = await rulesClient.update({ + id: existingRule.id, + data: convertRuleResponseToAlertingRule(ruleWithUpdates, actionsClient), + }); + return convertAlertingRuleToRuleResponse(updatedRule); + } + + const ruleWithDefaults = applyRuleDefaults(ruleToImport); + ruleWithDefaults.rule_source = ruleSource; + ruleWithDefaults.immutable = immutable; + + const payload = { + ...convertRuleResponseToAlertingRule(ruleWithDefaults, actionsClient), + alertTypeId: ruleTypeMappings[ruleToImport.type], + consumer: SERVER_APP_ID, + enabled: ruleToImport.enabled ?? false, + }; + + const createdRule = await rulesClient.create({ + data: payload, + options: { id: undefined }, + allowMissingConnectorSecrets, + }); + + return convertAlertingRuleToRuleResponse(createdRule); +}; From 899c31823af74574ddc51d6cac88b439fe08132a Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 10 Sep 2024 02:54:56 +0000 Subject: [PATCH 049/120] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../methods/import_rule_with_source.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts index 3f47d7a27f6d9..13c507d1570a9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts @@ -290,7 +290,7 @@ describe('importRuleWithSource', () => { expect(mockRulesClient.create).toHaveBeenCalledWith( expect.objectContaining({ data: expect.objectContaining({ - enabled: true + enabled: true, }), }) ); From c9ba72be58387479d6269153c2b5bcddd3ce591e Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 10 Sep 2024 14:56:58 -0500 Subject: [PATCH 050/120] Revert to version field being optional on import Sinec we use our RuleToImport type to parse individual rules on import, we can't make `version` required on the schema without that being a breaking change to existing systems. Instead, we can move the check to the new route, within the FF, so that versionless rules continue to function. --- .../import_rules/rule_to_import.ts | 2 -- .../logic/import/import_rules_utils.ts | 19 +++++++++++++++++++ .../import_rules.ts | 18 ++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index 66b18aee6c2d6..56b3b957f8508 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -12,7 +12,6 @@ import { RequiredFieldInput, RuleSignatureId, TypeSpecificCreateProps, - RuleVersion, } from '../../model/rule_schema'; /** @@ -31,7 +30,6 @@ export type RuleToImportInput = z.input; export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( ResponseFields.partial().extend({ rule_id: RuleSignatureId, - version: RuleVersion, /* Overriding `required_fields` from ResponseFields because in ResponseFields `required_fields` has the output type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index e8a1effeed2b2..510ae5a699526 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -120,6 +120,25 @@ export const importRules = async ({ } if (allowPrebuiltRules) { + if (!parsedRule.version) { + resolve( + createBulkErrorObject({ + statusCode: 400, + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.cannotImportRuleWithoutVersion', + { + defaultMessage: + 'Rules must specify a "version" to be imported. [rule_id: {ruleId}]', + values: { ruleId: parsedRule.rule_id }, + } + ), + ruleId: parsedRule.rule_id, + }) + ); + + return null; + } + const { immutable, ruleSource } = calculateRuleSourceForImport({ rule: parsedRule, prebuiltRuleAssets, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 1b9cc4af10020..b27187488cd73 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1683,6 +1683,24 @@ export default ({ getService }: FtrProviderContext): void => { }); }); + it('rejects rules without a version', async () => { + const rule = getCustomQueryRuleParams({}); + delete rule.version; + const ndjson = combineToNdJson(rule); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body.errors).toHaveLength(1); + expect(body.errors[0]).toMatchObject({ + error: { message: ': Required', status_code: 400 }, + }); + }); + describe('calculation of the rule_source fields', () => { it('calculates a version of 1 for custom rules'); From 49f70688d1bb02d63700cb5ad2c80b53f81ea564 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 10 Sep 2024 21:19:03 -0500 Subject: [PATCH 051/120] Update schema tests following making of 'version' optional --- .../import_rules/rule_to_import.test.ts | 51 +++++-------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts index 537c573c29e8b..8e7415b7c2729 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts @@ -22,7 +22,7 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 2 more"` + `"name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 1 more"` ); }); @@ -47,7 +47,7 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 1 more"` + `"name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql'"` ); }); @@ -61,7 +61,7 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"name: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', version: Required"` + `"name: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql'"` ); }); @@ -76,7 +76,7 @@ describe('RuleToImport', () => { expectParseError(result); expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"name: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', version: Required"` + `"name: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql'"` ); }); @@ -90,7 +90,6 @@ describe('RuleToImport', () => { severity: 'low', interval: '5m', type: 'query', - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -99,17 +98,6 @@ describe('RuleToImport', () => { expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"risk_score: Required"`); }); - test('missing version is not valid', () => { - const payload: RuleToImportInput = getImportRulesSchemaMock(); - // @ts-expect-error version is normally required - delete payload.version; - - const result = RuleToImport.safeParse(payload); - expectParseError(result); - - expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"version: Required"`); - }); - test('[rule_id, description, from, to, name, severity, type, interval, index] does not validate', () => { const payload: Partial = { rule_id: 'rule-1', @@ -121,7 +109,6 @@ describe('RuleToImport', () => { type: 'query', interval: '5m', index: ['index-1'], - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -130,7 +117,7 @@ describe('RuleToImport', () => { expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"risk_score: Required"`); }); - test('[rule_id, description, from, to, name, severity, type, query, index, interval, version] does validate', () => { + test('[rule_id, description, from, to, name, severity, type, query, index, interval] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', risk_score: 50, @@ -143,7 +130,6 @@ describe('RuleToImport', () => { query: 'some query', index: ['index-1'], interval: '5m', - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -151,7 +137,7 @@ describe('RuleToImport', () => { expectParseSuccess(result); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, version] does not validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, query, language] does not validate', () => { const payload: Partial = { rule_id: 'rule-1', description: 'some description', @@ -164,7 +150,6 @@ describe('RuleToImport', () => { type: 'query', query: 'some query', language: 'kuery', - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -173,7 +158,7 @@ describe('RuleToImport', () => { expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"risk_score: Required"`); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score, version] does validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', risk_score: 50, @@ -187,7 +172,6 @@ describe('RuleToImport', () => { type: 'query', query: 'some query', language: 'kuery', - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -195,7 +179,7 @@ describe('RuleToImport', () => { expectParseSuccess(result); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score, output_index, version] does validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score, output_index] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', output_index: '.siem-signals', @@ -210,7 +194,6 @@ describe('RuleToImport', () => { type: 'query', query: 'some query', language: 'kuery', - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -230,7 +213,6 @@ describe('RuleToImport', () => { interval: '5m', type: 'query', risk_score: 50, - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -251,7 +233,6 @@ describe('RuleToImport', () => { severity: 'low', interval: '5m', type: 'query', - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -280,7 +261,6 @@ describe('RuleToImport', () => { severity: 'low', interval: '5m', type: 'query', - version: 1, threat: [ { framework: 'someFramework', @@ -900,7 +880,7 @@ describe('RuleToImport', () => { ); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and version] does validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', description: 'some description', @@ -913,7 +893,6 @@ describe('RuleToImport', () => { type: 'query', risk_score: 50, note: '# some markdown', - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -922,7 +901,7 @@ describe('RuleToImport', () => { }); describe('exception_list', () => { - test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, version, and exceptions_list] does validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and exceptions_list] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', description: 'some description', @@ -937,14 +916,13 @@ describe('RuleToImport', () => { risk_score: 50, note: '# some markdown', exceptions_list: getListArrayMock(), - version: 1, }; const result = RuleToImport.safeParse(payload); expectParseSuccess(result); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, version, and empty exceptions_list] does validate', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, note, and empty exceptions_list] does validate', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', description: 'some description', @@ -959,7 +937,6 @@ describe('RuleToImport', () => { risk_score: 50, note: '# some markdown', exceptions_list: [], - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -980,7 +957,6 @@ describe('RuleToImport', () => { filters: [], risk_score: 50, note: '# some markdown', - version: 1, exceptions_list: [{ id: 'uuid_here', namespace_type: 'not a namespace type' }], }; @@ -992,7 +968,7 @@ describe('RuleToImport', () => { ); }); - test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, version, and non-existent exceptions_list] does validate with empty exceptions_list', () => { + test('[rule_id, description, from, to, index, name, severity, interval, type, filters, risk_score, note, and non-existent exceptions_list] does validate with empty exceptions_list', () => { const payload: RuleToImportInput = { rule_id: 'rule-1', description: 'some description', @@ -1006,7 +982,6 @@ describe('RuleToImport', () => { filters: [], risk_score: 50, note: '# some markdown', - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -1037,7 +1012,6 @@ describe('RuleToImport', () => { data_view_id: 'logs-*', index: [], interval: '5m', - version: 1, }; const result = RuleToImport.safeParse(payload); @@ -1059,7 +1033,6 @@ describe('RuleToImport', () => { data_view_id: 'logs-*', index: ['auditbeat-*'], interval: '5m', - version: 1, }; const result = RuleToImport.safeParse(payload); From 318da503d0bf928f65fabe6a42e196c4e37467ae Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 10 Sep 2024 21:50:12 -0500 Subject: [PATCH 052/120] Rename both our helper class and its methods This thing doesn't do the importing, but rather provides the information necessary for calculating prebuilt rule fields during import. --- .../prebuilt_rules_import_helper.mock.ts | 19 +++++++++ ...s => prebuilt_rules_import_helper.test.ts} | 42 +++++++++---------- ...ter.ts => prebuilt_rules_import_helper.ts} | 25 ++++++----- .../logic/prebuilt_rules_importer.mock.ts | 19 --------- .../api/rules/import_rules/route.ts | 6 +-- .../logic/import/import_rules_utils.test.ts | 22 +++++----- .../logic/import/import_rules_utils.ts | 12 +++--- 7 files changed, 75 insertions(+), 70 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.mock.ts rename x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/{prebuilt_rules_importer.test.ts => prebuilt_rules_import_helper.test.ts} (75%) rename x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/{prebuilt_rules_importer.ts => prebuilt_rules_import_helper.ts} (78%) delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.mock.ts new file mode 100644 index 0000000000000..9b14987d82519 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.mock.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PrebuiltRulesImportHelper } from './prebuilt_rules_import_helper'; + +const createPrebuiltRulesImportHelperMock = (): jest.Mocked => + ({ + setup: jest.fn(), + fetchMatchingAssets: jest.fn(), + fetchAssetRuleIds: jest.fn(), + } as unknown as jest.Mocked); + +export const prebuiltRulesImportHelperMock = { + create: createPrebuiltRulesImportHelperMock, +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.test.ts similarity index 75% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.test.ts index 23405a1609cde..515b880079acc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.test.ts @@ -11,7 +11,7 @@ import type { RuleToImport } from '../../../../../common/api/detection_engine'; import { createPrebuiltRuleAssetsClient as createPrebuiltRuleAssetsClientMock } from './rule_assets/__mocks__/prebuilt_rule_assets_client'; import { ensureLatestRulesPackageInstalled } from './ensure_latest_rules_package_installed'; import { configMock, createMockConfig, requestContextMock } from '../../routes/__mocks__'; -import { PrebuiltRulesImporter } from './prebuilt_rules_importer'; +import { PrebuiltRulesImportHelper } from './prebuilt_rules_import_helper'; jest.mock('./ensure_latest_rules_package_installed'); @@ -21,7 +21,7 @@ jest.mock('./rule_assets/prebuilt_rule_assets_client', () => ({ createPrebuiltRuleAssetsClient: () => mockPrebuiltRuleAssetsClient, })); -describe('PrebuiltRulesImporter', () => { +describe('PrebuiltRulesImportHelper', () => { let config: ReturnType; let context: ReturnType['securitySolution']; let savedObjectsClient: jest.Mocked; @@ -39,7 +39,7 @@ describe('PrebuiltRulesImporter', () => { }); it('should initialize correctly', () => { - const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); expect(importer).toBeDefined(); expect(importer.enabled).toBe(true); @@ -49,7 +49,7 @@ describe('PrebuiltRulesImporter', () => { describe('setup', () => { it('should not call ensureLatestRulesPackageInstalled if disabled', async () => { config = createMockConfig(); - const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); await importer.setup(); @@ -58,7 +58,7 @@ describe('PrebuiltRulesImporter', () => { }); it('should call ensureLatestRulesPackageInstalled if enabled', async () => { - const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); await importer.setup(); @@ -71,26 +71,26 @@ describe('PrebuiltRulesImporter', () => { }); }); - describe('fetchPrebuiltRuleAssets', () => { + describe('fetchMatchingAssets', () => { it('should return an empty array if disabled', async () => { config = createMockConfig(); - const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - const result = await importer.fetchPrebuiltRuleAssets({ rules: [] }); + const result = await importer.fetchMatchingAssets({ rules: [] }); expect(result).toEqual([]); }); it('should throw an error if latestPackagesInstalled is false', async () => { - const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - await expect(importer.fetchPrebuiltRuleAssets({ rules: [] })).rejects.toThrow( + await expect(importer.fetchMatchingAssets({ rules: [] })).rejects.toThrow( 'Prebuilt rule assets cannot be fetched until the latest rules package is installed. Call setup() on this object first.' ); }); it('should fetch prebuilt rule assets correctly if latestPackagesInstalled is true', async () => { - const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); importer.latestPackagesInstalled = true; const rules = [ @@ -99,7 +99,7 @@ describe('PrebuiltRulesImporter', () => { new Error('Invalid rule'), ] as Array; - await importer.fetchPrebuiltRuleAssets({ rules }); + await importer.fetchMatchingAssets({ rules }); expect(mockPrebuiltRuleAssetsClient.fetchAssetsByVersion).toHaveBeenCalledWith([ { rule_id: 'rule-1', version: 1 }, @@ -108,27 +108,27 @@ describe('PrebuiltRulesImporter', () => { }); }); - describe('fetchInstalledRuleIds', () => { + describe('fetchAssetRuleIds', () => { it('returns an empty array if the importer is not enabled', async () => { - const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); importer.enabled = false; - const result = await importer.fetchInstalledRuleIds({ rules: [] }); + const result = await importer.fetchAssetRuleIds({ rules: [] }); expect(result).toEqual([]); }); it('throws an error if the latest packages are not installed', async () => { - const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); importer.latestPackagesInstalled = false; - await expect(importer.fetchInstalledRuleIds({ rules: [] })).rejects.toThrow( + await expect(importer.fetchAssetRuleIds({ rules: [] })).rejects.toThrow( 'Installed rule IDs cannot be fetched until the latest rules package is installed. Call setup() on this object first.' ); }); it('fetches and return the rule IDs of installed prebuilt rules', async () => { - const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); importer.latestPackagesInstalled = true; const rules = [ { rule_id: 'rule-1', version: 1 }, @@ -141,7 +141,7 @@ describe('PrebuiltRulesImporter', () => { installedRuleAssets ); - const result = await importer.fetchInstalledRuleIds({ rules }); + const result = await importer.fetchAssetRuleIds({ rules }); expect(mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId).toHaveBeenCalledWith([ 'rule-1', @@ -151,13 +151,13 @@ describe('PrebuiltRulesImporter', () => { }); it('handles rules that are instances of Error', async () => { - const importer = new PrebuiltRulesImporter({ config, context, savedObjectsClient }); + const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); importer.latestPackagesInstalled = true; const rules = [new Error('Invalid rule')]; (mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId as jest.Mock).mockResolvedValue([]); - const result = await importer.fetchInstalledRuleIds({ rules }); + const result = await importer.fetchAssetRuleIds({ rules }); expect(mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId).toHaveBeenCalledWith([]); expect(result).toEqual([]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.ts similarity index 78% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.ts index 33a1e496d5502..79ce18f407de7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.ts @@ -14,16 +14,21 @@ import { createPrebuiltRuleAssetsClient } from './rule_assets/prebuilt_rule_asse import { ensureLatestRulesPackageInstalled } from './ensure_latest_rules_package_installed'; type MaybeRule = RuleToImport | Error; -export interface IPrebuiltRulesImporter { + +export interface IPrebuiltRulesImportHelper { setup: () => Promise; - fetchPrebuiltRuleAssets: ({ rules }: { rules: MaybeRule[] }) => Promise; + fetchMatchingAssets: ({ rules }: { rules: MaybeRule[] }) => Promise; + fetchAssetRuleIds: ({ rules }: { rules: MaybeRule[] }) => Promise; } /** * - * This object contains logic necessary to import prebuilt rules into the system. + * This class contains utilities for assisting with the importing of prebuilt + * rules. It ensures that the system contains the necessary assets, and also + * provides utilities for fetching information from them, necessary for + * importing prebuilt rules. */ -export class PrebuiltRulesImporter implements IPrebuiltRulesImporter { +export class PrebuiltRulesImportHelper implements IPrebuiltRulesImportHelper { private context: SecuritySolutionApiRequestHandlerContext; private config: ConfigType; private ruleAssetsClient: ReturnType; @@ -67,7 +72,7 @@ export class PrebuiltRulesImporter implements IPrebuiltRulesImporter { * rules, which are used to determine how to import those rules (create vs. update, etc.). * Assets match the `rule_id` and `version` of the specified rules. */ - public async fetchPrebuiltRuleAssets({ + public async fetchMatchingAssets({ rules, }: { rules: MaybeRule[]; @@ -96,16 +101,16 @@ export class PrebuiltRulesImporter implements IPrebuiltRulesImporter { } /** - * Retrieves the rule IDs (`rule_id`s) of installed prebuilt rules matching those - * of the specified rules. Can be used to determine whether the rule being - * imported is a custom rule or a prebuilt rule. + * Retrieves the rule IDs (`rule_id`s) of available prebuilt rule assets matching those + * of the specified rules. This information can be used to determine whether + * the rule being imported is a custom rule or a prebuilt rule. * * @param rules - A list of rules being imported. * - * @returns A list of the rule IDs that are installed. + * @returns A list of the prebuilt rule IDs that are available. * */ - public async fetchInstalledRuleIds({ rules }: { rules: MaybeRule[] }): Promise { + public async fetchAssetRuleIds({ rules }: { rules: MaybeRule[] }): Promise { if (!this.enabled) { return []; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts deleted file mode 100644 index d96d0bc0977eb..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_importer.mock.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { PrebuiltRulesImporter } from './prebuilt_rules_importer'; - -const createPrebuiltRulesImporterMock = (): jest.Mocked => - ({ - setup: jest.fn(), - fetchPrebuiltRuleAssets: jest.fn(), - fetchInstalledRuleIds: jest.fn(), - } as unknown as jest.Mocked); - -export const prebuiltRulesImporterMock = { - create: createPrebuiltRulesImporterMock, -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 0fca6a104f13c..efeba4172dfb8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -21,7 +21,7 @@ import type { ConfigType } from '../../../../../../config'; import type { HapiReadableStream, SecuritySolutionPluginRouter } from '../../../../../../types'; import type { BulkError, ImportRuleResponse } from '../../../../routes/utils'; import { buildSiemResponse, isBulkError, isImportRegular } from '../../../../routes/utils'; -import { PrebuiltRulesImporter } from '../../../../prebuilt_rules/logic/prebuilt_rules_importer'; +import { PrebuiltRulesImportHelper } from '../../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; import { createRulesAndExceptionsStreamFromNdJson } from '../../../logic/import/create_rules_stream_from_ndjson'; import type { RuleExceptionsPromiseFromStreams } from '../../../logic/import/import_rules_utils'; @@ -144,7 +144,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C ? [] : rulesWithMigratedActions || migratedParsedObjectsWithoutDuplicateErrors; - const prebuiltRulesImporter = new PrebuiltRulesImporter({ + const prebuiltRulesImportHelper = new PrebuiltRulesImportHelper({ config, context: ctx.securitySolution, savedObjectsClient, @@ -158,7 +158,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C overwriteRules: request.query.overwrite, detectionRulesClient, allowMissingConnectorSecrets: !!actionConnectors.length, - prebuiltRulesImporter, + prebuiltRulesImportHelper, allowPrebuiltRules: prebuiltRulesCustomizationEnabled, savedObjectsClient, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts index d039c8b715516..db3d4cdbc1a0c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts @@ -10,7 +10,7 @@ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; import { requestContextMock } from '../../../routes/__mocks__'; -import { prebuiltRulesImporterMock } from '../../../prebuilt_rules/logic/prebuilt_rules_importer.mock'; +import { prebuiltRulesImportHelperMock } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper.mock'; import { importRules } from './import_rules_utils'; import { createBulkErrorObject } from '../../../routes/utils'; @@ -20,15 +20,15 @@ describe('importRules', () => { const ruleToImport = getImportRulesSchemaMock(); let savedObjectsClient: jest.Mocked; - let mockPrebuiltRulesImporter: ReturnType; + let mockPrebuiltRulesImportHelper: ReturnType; beforeEach(() => { jest.clearAllMocks(); savedObjectsClient = savedObjectsClientMock.create(); - mockPrebuiltRulesImporter = prebuiltRulesImporterMock.create(); - mockPrebuiltRulesImporter.fetchPrebuiltRuleAssets.mockResolvedValue([]); - mockPrebuiltRulesImporter.fetchInstalledRuleIds.mockResolvedValue([]); + mockPrebuiltRulesImportHelper = prebuiltRulesImportHelperMock.create(); + mockPrebuiltRulesImportHelper.fetchMatchingAssets.mockResolvedValue([]); + mockPrebuiltRulesImportHelper.fetchAssetRuleIds.mockResolvedValue([]); }); it('returns an empty rules response if no rules to import', async () => { @@ -38,7 +38,7 @@ describe('importRules', () => { overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), savedObjectsClient, - prebuiltRulesImporter: mockPrebuiltRulesImporter, + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, }); expect(result).toEqual([]); @@ -50,7 +50,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImporter: mockPrebuiltRulesImporter, + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, savedObjectsClient, }); @@ -80,7 +80,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImporter: mockPrebuiltRulesImporter, + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, savedObjectsClient, }); @@ -107,7 +107,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImporter: mockPrebuiltRulesImporter, + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, savedObjectsClient, }); @@ -132,7 +132,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImporter: mockPrebuiltRulesImporter, + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, allowPrebuiltRules: false, savedObjectsClient, }); @@ -161,7 +161,7 @@ describe('importRules', () => { overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), allowPrebuiltRules: true, - prebuiltRulesImporter: mockPrebuiltRulesImporter, + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, savedObjectsClient, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 510ae5a699526..bfabe28c0df9f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -15,7 +15,7 @@ import type { import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; -import type { PrebuiltRulesImporter } from '../../../prebuilt_rules/logic/prebuilt_rules_importer'; +import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; import { checkRuleExceptionReferences } from './check_rule_exception_references'; import { calculateRuleSourceForImport } from './calculate_rule_source_for_import'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; @@ -47,7 +47,7 @@ export const importRules = async ({ rulesResponseAcc, overwriteRules, detectionRulesClient, - prebuiltRulesImporter, + prebuiltRulesImportHelper, allowPrebuiltRules, allowMissingConnectorSecrets, savedObjectsClient, @@ -56,7 +56,7 @@ export const importRules = async ({ rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; - prebuiltRulesImporter: PrebuiltRulesImporter; + prebuiltRulesImportHelper: PrebuiltRulesImportHelper; allowPrebuiltRules?: boolean; allowMissingConnectorSecrets?: boolean; savedObjectsClient: SavedObjectsClientContract; @@ -69,7 +69,7 @@ export const importRules = async ({ return importRuleResponse; } - await prebuiltRulesImporter.setup(); + await prebuiltRulesImportHelper.setup(); while (ruleChunks.length) { const batchParseObjects = ruleChunks.shift() ?? []; @@ -77,10 +77,10 @@ export const importRules = async ({ rules: batchParseObjects, savedObjectsClient, }); - const prebuiltRuleAssets = await prebuiltRulesImporter.fetchPrebuiltRuleAssets({ + const prebuiltRuleAssets = await prebuiltRulesImportHelper.fetchMatchingAssets({ rules: batchParseObjects, }); - const installedRuleIds = await prebuiltRulesImporter.fetchInstalledRuleIds({ + const installedRuleIds = await prebuiltRulesImportHelper.fetchAssetRuleIds({ rules: batchParseObjects, }); From 937ecc9e57ff197c7b5c2a494fa1395fa467a1f6 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 10 Sep 2024 22:20:57 -0500 Subject: [PATCH 053/120] Move importRules interface into DetectionRulesClient Since this class already contains the `importRule` method (which was a thin wrapper around the pure `importRule` function), it makes sense to do the same for the outer loop. --- .../api/rules/import_rules/route.ts | 4 +--- .../detection_rules_client.ts | 12 ++++++++++++ .../detection_rules_client_interface.ts | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index efeba4172dfb8..0989e5bd31f35 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -25,7 +25,6 @@ import { PrebuiltRulesImportHelper } from '../../../../prebuilt_rules/logic/preb import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; import { createRulesAndExceptionsStreamFromNdJson } from '../../../logic/import/create_rules_stream_from_ndjson'; import type { RuleExceptionsPromiseFromStreams } from '../../../logic/import/import_rules_utils'; -import { importRules as importRulesHelper } from '../../../logic/import/import_rules_utils'; import { importRuleExceptions } from '../../../logic/import/import_rule_exceptions'; import { getTupleDuplicateErrorsAndUniqueRules, @@ -152,11 +151,10 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C const chunkParseObjects = chunk(CHUNK_PARSED_OBJECT_SIZE, parsedRules); - const importRuleResponse: ImportRuleResponse[] = await importRulesHelper({ + const importRuleResponse: ImportRuleResponse[] = await detectionRulesClient.importRules({ ruleChunks: chunkParseObjects, rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], overwriteRules: request.query.overwrite, - detectionRulesClient, allowMissingConnectorSecrets: !!actionConnectors.length, prebuiltRulesImportHelper, allowPrebuiltRules: prebuiltRulesCustomizationEnabled, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts index 4753df0ffe411..194274a7647ba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts @@ -13,12 +13,15 @@ import type { RuleResponse } from '../../../../../../common/api/detection_engine import { withSecuritySpan } from '../../../../../utils/with_security_span'; import type { MlAuthz } from '../../../../machine_learning/authz'; import { createPrebuiltRuleAssetsClient } from '../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; +import type { ImportRuleResponse } from '../../../routes/utils'; +import { importRules } from '../import/import_rules_utils'; import type { CreateCustomRuleArgs, CreatePrebuiltRuleArgs, DeleteRuleArgs, IDetectionRulesClient, ImportRuleArgs, + ImportRulesArgs, PatchRuleArgs, UpdateRuleArgs, UpgradePrebuiltRuleArgs, @@ -131,5 +134,14 @@ export const createDetectionRulesClient = ({ }); }); }, + + async importRules(args: ImportRulesArgs): Promise { + return withSecuritySpan('DetectionRulesClient.importRules', async () => { + return importRules({ + ...args, + detectionRulesClient: this, + }); + }); + }, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts index d7b45f83e8bf8..6c576a96b9c7b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { SavedObjectsClientContract } from '@kbn/core/server'; import type { RuleCreateProps, RuleUpdateProps, @@ -14,6 +15,9 @@ import type { RuleResponse, } from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; +import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; +import type { ImportRuleResponse } from '../../../routes/utils'; +import type { PromiseFromStreams } from '../import/import_rules_utils'; export interface IDetectionRulesClient { createCustomRule: (args: CreateCustomRuleArgs) => Promise; @@ -23,6 +27,7 @@ export interface IDetectionRulesClient { deleteRule: (args: DeleteRuleArgs) => Promise; upgradePrebuiltRule: (args: UpgradePrebuiltRuleArgs) => Promise; importRule: (args: ImportRuleArgs) => Promise; + importRules: (args: ImportRulesArgs) => Promise; } export interface CreateCustomRuleArgs { @@ -54,3 +59,13 @@ export interface ImportRuleArgs { overwriteRules?: boolean; allowMissingConnectorSecrets?: boolean; } + +export interface ImportRulesArgs { + ruleChunks: PromiseFromStreams[][]; + rulesResponseAcc: ImportRuleResponse[]; + overwriteRules: boolean; + prebuiltRulesImportHelper: PrebuiltRulesImportHelper; + allowPrebuiltRules?: boolean; + allowMissingConnectorSecrets?: boolean; + savedObjectsClient: SavedObjectsClientContract; +} From 9f3b9e5f68d7052874a015586bda91e5b2b80f3f Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 10 Sep 2024 22:54:51 -0500 Subject: [PATCH 054/120] WIP: Adding secondary code paths for new 'withSource' implementations This moves the switching on feature flag to the highest level possible: the route. Based on that determination, we call either the new `detectionRulesClient.importRules` method, or the old `importRules` function. There are some type errors related to some types still being shared between the implementations, and I need to verify that the new tests are in the right place, but as of this commit things are plumbed through and are mostly in the right place. --- .../api/rules/import_rules/route.ts | 31 +++- .../detection_rules_client.ts | 17 +- .../detection_rules_client_interface.ts | 2 +- .../logic/import/import_rules_utils.test.ts | 77 ++------ .../logic/import/import_rules_utils.ts | 49 +---- .../import/import_rules_with_source.test.ts | 148 +++++++++++++++ .../logic/import/import_rules_with_source.ts | 172 ++++++++++++++++++ 7 files changed, 379 insertions(+), 117 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 0989e5bd31f35..c5b996fe4a97b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -23,6 +23,7 @@ import type { BulkError, ImportRuleResponse } from '../../../../routes/utils'; import { buildSiemResponse, isBulkError, isImportRegular } from '../../../../routes/utils'; import { PrebuiltRulesImportHelper } from '../../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; +import { importRules as legacyImportRules } from '../../../logic/import/import_rules_utils'; import { createRulesAndExceptionsStreamFromNdJson } from '../../../logic/import/create_rules_stream_from_ndjson'; import type { RuleExceptionsPromiseFromStreams } from '../../../logic/import/import_rules_utils'; import { importRuleExceptions } from '../../../logic/import/import_rule_exceptions'; @@ -151,15 +152,27 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C const chunkParseObjects = chunk(CHUNK_PARSED_OBJECT_SIZE, parsedRules); - const importRuleResponse: ImportRuleResponse[] = await detectionRulesClient.importRules({ - ruleChunks: chunkParseObjects, - rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], - overwriteRules: request.query.overwrite, - allowMissingConnectorSecrets: !!actionConnectors.length, - prebuiltRulesImportHelper, - allowPrebuiltRules: prebuiltRulesCustomizationEnabled, - savedObjectsClient, - }); + let importRuleResponse: ImportRuleResponse[] = []; + + if (prebuiltRulesCustomizationEnabled) { + importRuleResponse = await detectionRulesClient.importRules({ + ruleChunks: chunkParseObjects, + rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], + overwriteRules: request.query.overwrite, + allowMissingConnectorSecrets: !!actionConnectors.length, + prebuiltRulesImportHelper, + savedObjectsClient, + }); + } else { + importRuleResponse = await legacyImportRules({ + ruleChunks: chunkParseObjects, + rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], + overwriteRules: request.query.overwrite, + allowMissingConnectorSecrets: !!actionConnectors.length, + detectionRulesClient, + savedObjectsClient, + }); + } const errorsResp = importRuleResponse.filter((resp) => isBulkError(resp)) as BulkError[]; const successes = importRuleResponse.filter((resp) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts index 194274a7647ba..a1d65e5bbf6ce 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts @@ -14,7 +14,7 @@ import { withSecuritySpan } from '../../../../../utils/with_security_span'; import type { MlAuthz } from '../../../../machine_learning/authz'; import { createPrebuiltRuleAssetsClient } from '../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import type { ImportRuleResponse } from '../../../routes/utils'; -import { importRules } from '../import/import_rules_utils'; +import { importRules } from '../import/import_rules_with_source'; import type { CreateCustomRuleArgs, CreatePrebuiltRuleArgs, @@ -32,6 +32,7 @@ import { importRule } from './methods/import_rule'; import { patchRule } from './methods/patch_rule'; import { updateRule } from './methods/update_rule'; import { upgradePrebuiltRule } from './methods/upgrade_prebuilt_rule'; +import { importRuleWithSource } from './methods/import_rule_with_source'; interface DetectionRulesClientParams { actionsClient: ActionsClient; @@ -123,9 +124,21 @@ export const createDetectionRulesClient = ({ }); }, + async legacyImportRule(args: ImportRuleArgs): Promise { + return withSecuritySpan('DetectionRulesClient.legacyImportRule', async () => { + return importRule({ + actionsClient, + rulesClient, + importRulePayload: args, + mlAuthz, + prebuiltRuleAssetClient, + }); + }); + }, + async importRule(args: ImportRuleArgs): Promise { return withSecuritySpan('DetectionRulesClient.importRule', async () => { - return importRule({ + return importRuleWithSource({ actionsClient, rulesClient, importRulePayload: args, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts index 6c576a96b9c7b..c3b2bf4862f8e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts @@ -26,6 +26,7 @@ export interface IDetectionRulesClient { patchRule: (args: PatchRuleArgs) => Promise; deleteRule: (args: DeleteRuleArgs) => Promise; upgradePrebuiltRule: (args: UpgradePrebuiltRuleArgs) => Promise; + legacyImportRule: (args: ImportRuleArgs) => Promise; importRule: (args: ImportRuleArgs) => Promise; importRules: (args: ImportRulesArgs) => Promise; } @@ -65,7 +66,6 @@ export interface ImportRulesArgs { rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; prebuiltRulesImportHelper: PrebuiltRulesImportHelper; - allowPrebuiltRules?: boolean; allowMissingConnectorSecrets?: boolean; savedObjectsClient: SavedObjectsClientContract; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts index db3d4cdbc1a0c..5a582c28fda78 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts @@ -10,7 +10,6 @@ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; import { requestContextMock } from '../../../routes/__mocks__'; -import { prebuiltRulesImportHelperMock } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper.mock'; import { importRules } from './import_rules_utils'; import { createBulkErrorObject } from '../../../routes/utils'; @@ -20,15 +19,11 @@ describe('importRules', () => { const ruleToImport = getImportRulesSchemaMock(); let savedObjectsClient: jest.Mocked; - let mockPrebuiltRulesImportHelper: ReturnType; beforeEach(() => { jest.clearAllMocks(); savedObjectsClient = savedObjectsClientMock.create(); - mockPrebuiltRulesImportHelper = prebuiltRulesImportHelperMock.create(); - mockPrebuiltRulesImportHelper.fetchMatchingAssets.mockResolvedValue([]); - mockPrebuiltRulesImportHelper.fetchAssetRuleIds.mockResolvedValue([]); }); it('returns an empty rules response if no rules to import', async () => { @@ -38,7 +33,6 @@ describe('importRules', () => { overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), savedObjectsClient, - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, }); expect(result).toEqual([]); @@ -50,7 +44,6 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, savedObjectsClient, }); @@ -80,7 +73,6 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, savedObjectsClient, }); @@ -107,65 +99,34 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, savedObjectsClient, }); expect(result).toEqual([{ rule_id: ruleToImport.rule_id, status_code: 200 }]); }); - describe('compatibility with prebuilt rules', () => { - let prebuiltRuleToImport: ReturnType; - - beforeEach(() => { - prebuiltRuleToImport = { - ...getImportRulesSchemaMock(), - immutable: true, - version: 1, - }; + it('rejects a prebuilt rule specifying an immutable value of true', async () => { + const prebuiltRuleToImport = { + ...getImportRulesSchemaMock(), + immutable: true, + version: 1, + }; + const result = await importRules({ + ruleChunks: [[prebuiltRuleToImport]], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + savedObjectsClient, }); - it('rejects a prebuilt rule when allowPrebuiltRules is false', async () => { - const ruleChunk = [prebuiltRuleToImport]; - const result = await importRules({ - ruleChunks: [ruleChunk], - rulesResponseAcc: [], - overwriteRules: false, - detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, - allowPrebuiltRules: false, - savedObjectsClient, - }); - - expect(result).toEqual([ - { - error: { - message: `Importing prebuilt rules is not supported. To import this rule as a custom rule, first duplicate the rule and then export it. [rule_id: ${prebuiltRuleToImport.rule_id}]`, - status_code: 400, - }, - rule_id: prebuiltRuleToImport.rule_id, + expect(result).toEqual([ + { + error: { + message: `Importing prebuilt rules is not supported. To import this rule as a custom rule, first duplicate the rule and then export it. [rule_id: ${prebuiltRuleToImport.rule_id}]`, + status_code: 400, }, - ]); - }); - - it('imports a prebuilt rule when allowPrebuiltRules is true', async () => { - clients.detectionRulesClient.importRule.mockResolvedValue({ - ...getRulesSchemaMock(), rule_id: prebuiltRuleToImport.rule_id, - }); - - const ruleChunk = [prebuiltRuleToImport]; - const result = await importRules({ - ruleChunks: [ruleChunk], - rulesResponseAcc: [], - overwriteRules: false, - detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - allowPrebuiltRules: true, - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, - savedObjectsClient, - }); - - expect(result).toEqual([{ rule_id: prebuiltRuleToImport.rule_id, status_code: 200 }]); - }); + }, + ]); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index bfabe28c0df9f..9ce96239f753d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -15,9 +15,7 @@ import type { import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; -import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; import { checkRuleExceptionReferences } from './check_rule_exception_references'; -import { calculateRuleSourceForImport } from './calculate_rule_source_for_import'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { getReferencedExceptionLists } from './gather_referenced_exceptions'; @@ -47,8 +45,6 @@ export const importRules = async ({ rulesResponseAcc, overwriteRules, detectionRulesClient, - prebuiltRulesImportHelper, - allowPrebuiltRules, allowMissingConnectorSecrets, savedObjectsClient, }: { @@ -56,8 +52,6 @@ export const importRules = async ({ rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; - prebuiltRulesImportHelper: PrebuiltRulesImportHelper; - allowPrebuiltRules?: boolean; allowMissingConnectorSecrets?: boolean; savedObjectsClient: SavedObjectsClientContract; }) => { @@ -69,21 +63,12 @@ export const importRules = async ({ return importRuleResponse; } - await prebuiltRulesImportHelper.setup(); - while (ruleChunks.length) { const batchParseObjects = ruleChunks.shift() ?? []; const existingLists = await getReferencedExceptionLists({ rules: batchParseObjects, savedObjectsClient, }); - const prebuiltRuleAssets = await prebuiltRulesImportHelper.fetchMatchingAssets({ - rules: batchParseObjects, - }); - const installedRuleIds = await prebuiltRulesImportHelper.fetchAssetRuleIds({ - rules: batchParseObjects, - }); - const newImportRuleResponse = await Promise.all( batchParseObjects.reduce>>((accum, parsedRule) => { const importsWorkerPromise = new Promise(async (resolve, reject) => { @@ -100,7 +85,7 @@ export const importRules = async ({ return null; } - if (!allowPrebuiltRules && parsedRule.immutable) { + if (parsedRule.immutable) { resolve( createBulkErrorObject({ statusCode: 400, @@ -119,36 +104,6 @@ export const importRules = async ({ return null; } - if (allowPrebuiltRules) { - if (!parsedRule.version) { - resolve( - createBulkErrorObject({ - statusCode: 400, - message: i18n.translate( - 'xpack.securitySolution.detectionEngine.rules.cannotImportRuleWithoutVersion', - { - defaultMessage: - 'Rules must specify a "version" to be imported. [rule_id: {ruleId}]', - values: { ruleId: parsedRule.rule_id }, - } - ), - ruleId: parsedRule.rule_id, - }) - ); - - return null; - } - - const { immutable, ruleSource } = calculateRuleSourceForImport({ - rule: parsedRule, - prebuiltRuleAssets, - installedRuleIds, - }); - - parsedRule.rule_source = ruleSource; - parsedRule.immutable = immutable; - } - try { const [exceptionErrors, exceptions] = checkRuleExceptionReferences({ rule: parsedRule, @@ -157,7 +112,7 @@ export const importRules = async ({ importRuleResponse = [...importRuleResponse, ...exceptionErrors]; - const importedRule = await detectionRulesClient.importRule({ + const importedRule = await detectionRulesClient.legacyImportRule({ ruleToImport: { ...parsedRule, exceptions_list: [...exceptions], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts new file mode 100644 index 0000000000000..0c44d66d43c52 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from '@kbn/core/server'; +import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; +import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; +import { requestContextMock } from '../../../routes/__mocks__'; +import { prebuiltRulesImportHelperMock } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper.mock'; + +import { importRules } from './import_rules_with_source'; +import { createBulkErrorObject } from '../../../routes/utils'; + +describe('importRules', () => { + const { clients, context } = requestContextMock.createTools(); + const ruleToImport = getImportRulesSchemaMock(); + + let savedObjectsClient: jest.Mocked; + let mockPrebuiltRulesImportHelper: ReturnType; + + beforeEach(() => { + jest.clearAllMocks(); + + savedObjectsClient = savedObjectsClientMock.create(); + mockPrebuiltRulesImportHelper = prebuiltRulesImportHelperMock.create(); + mockPrebuiltRulesImportHelper.fetchMatchingAssets.mockResolvedValue([]); + mockPrebuiltRulesImportHelper.fetchAssetRuleIds.mockResolvedValue([]); + }); + + it('returns an empty rules response if no rules to import', async () => { + const result = await importRules({ + ruleChunks: [], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + savedObjectsClient, + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, + }); + + expect(result).toEqual([]); + }); + + it('returns 400 error if "ruleChunks" includes Error', async () => { + const result = await importRules({ + ruleChunks: [[new Error('error importing')]], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, + savedObjectsClient, + }); + + expect(result).toEqual([ + { + error: { + message: 'error importing', + status_code: 400, + }, + rule_id: '(unknown id)', + }, + ]); + }); + + it('returns 409 error if DetectionRulesClient throws with 409 - existing rule', async () => { + clients.detectionRulesClient.importRule.mockImplementationOnce(async () => { + throw createBulkErrorObject({ + ruleId: ruleToImport.rule_id, + statusCode: 409, + message: `rule_id: "${ruleToImport.rule_id}" already exists`, + }); + }); + + const ruleChunk = [ruleToImport]; + const result = await importRules({ + ruleChunks: [ruleChunk], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, + savedObjectsClient, + }); + + expect(result).toEqual([ + { + error: { + message: `rule_id: "${ruleToImport.rule_id}" already exists`, + status_code: 409, + }, + rule_id: ruleToImport.rule_id, + }, + ]); + }); + + it('creates rule if no matching existing rule found', async () => { + clients.detectionRulesClient.importRule.mockResolvedValue({ + ...getRulesSchemaMock(), + rule_id: ruleToImport.rule_id, + }); + + const ruleChunk = [ruleToImport]; + const result = await importRules({ + ruleChunks: [ruleChunk], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, + savedObjectsClient, + }); + + expect(result).toEqual([{ rule_id: ruleToImport.rule_id, status_code: 200 }]); + }); + + describe('compatibility with prebuilt rules', () => { + let prebuiltRuleToImport: ReturnType; + + beforeEach(() => { + prebuiltRuleToImport = { + ...getImportRulesSchemaMock(), + immutable: true, + version: 1, + }; + }); + + it('imports a prebuilt rule when allowPrebuiltRules is true', async () => { + clients.detectionRulesClient.importRule.mockResolvedValue({ + ...getRulesSchemaMock(), + rule_id: prebuiltRuleToImport.rule_id, + }); + + const ruleChunk = [prebuiltRuleToImport]; + const result = await importRules({ + ruleChunks: [ruleChunk], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient: context.securitySolution.getDetectionRulesClient(), + allowPrebuiltRules: true, + prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, + savedObjectsClient, + }); + + expect(result).toEqual([{ rule_id: prebuiltRuleToImport.rule_id, status_code: 200 }]); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts new file mode 100644 index 0000000000000..05470fbfd93a5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import type { SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; +import type { + ImportExceptionsListSchema, + ImportExceptionListItemSchema, +} from '@kbn/securitysolution-io-ts-list-types'; + +import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; +import type { ImportRuleResponse } from '../../../routes/utils'; +import { createBulkErrorObject } from '../../../routes/utils'; +import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; +import { checkRuleExceptionReferences } from './check_rule_exception_references'; +import { calculateRuleSourceForImport } from './calculate_rule_source_for_import'; +import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; +import { getReferencedExceptionLists } from './gather_referenced_exceptions'; + +export type PromiseFromStreams = RuleToImport | Error; +export interface RuleExceptionsPromiseFromStreams { + rules: PromiseFromStreams[]; + exceptions: Array; + actionConnectors: SavedObject[]; +} + +/** + * Takes rules to be imported and either creates or updates rules + * based on user overwrite preferences + * @param ruleChunks {array} - rules being imported + * @param rulesResponseAcc {array} - the accumulation of success and + * error messages gathered through the rules import logic + * @param mlAuthz {object} + * @param overwriteRules {boolean} - whether to overwrite existing rules + * with imported rules if their rule_id matches + * @param detectionRulesClient {object} + * @param existingLists {object} - all exception lists referenced by + * rules that were found to exist + * @returns {Promise} an array of error and success messages from import + */ +export const importRules = async ({ + ruleChunks, + rulesResponseAcc, + overwriteRules, + detectionRulesClient, + prebuiltRulesImportHelper, + allowPrebuiltRules, + allowMissingConnectorSecrets, + savedObjectsClient, +}: { + ruleChunks: PromiseFromStreams[][]; + rulesResponseAcc: ImportRuleResponse[]; + overwriteRules: boolean; + detectionRulesClient: IDetectionRulesClient; + prebuiltRulesImportHelper: PrebuiltRulesImportHelper; + allowPrebuiltRules?: boolean; + allowMissingConnectorSecrets?: boolean; + savedObjectsClient: SavedObjectsClientContract; +}) => { + let importRuleResponse: ImportRuleResponse[] = [...rulesResponseAcc]; + + // If we had 100% errors and no successful rule could be imported we still have to output an error. + // otherwise we would output we are success importing 0 rules. + if (ruleChunks.length === 0) { + return importRuleResponse; + } + + await prebuiltRulesImportHelper.setup(); + + while (ruleChunks.length) { + const batchParseObjects = ruleChunks.shift() ?? []; + const existingLists = await getReferencedExceptionLists({ + rules: batchParseObjects, + savedObjectsClient, + }); + const prebuiltRuleAssets = await prebuiltRulesImportHelper.fetchMatchingAssets({ + rules: batchParseObjects, + }); + const installedRuleIds = await prebuiltRulesImportHelper.fetchAssetRuleIds({ + rules: batchParseObjects, + }); + + const newImportRuleResponse = await Promise.all( + batchParseObjects.reduce>>((accum, parsedRule) => { + const importsWorkerPromise = new Promise(async (resolve, reject) => { + try { + if (parsedRule instanceof Error) { + // If the JSON object had a validation or parse error then we return + // early with the error and an (unknown) for the ruleId + resolve( + createBulkErrorObject({ + statusCode: 400, + message: parsedRule.message, + }) + ); + return null; + } + + if (!parsedRule.version) { + resolve( + createBulkErrorObject({ + statusCode: 400, + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.cannotImportRuleWithoutVersion', + { + defaultMessage: + 'Rules must specify a "version" to be imported. [rule_id: {ruleId}]', + values: { ruleId: parsedRule.rule_id }, + } + ), + ruleId: parsedRule.rule_id, + }) + ); + + return null; + } + + const { immutable, ruleSource } = calculateRuleSourceForImport({ + rule: parsedRule, + prebuiltRuleAssets, + installedRuleIds, + }); + parsedRule.rule_source = ruleSource; + parsedRule.immutable = immutable; + + try { + const [exceptionErrors, exceptions] = checkRuleExceptionReferences({ + rule: parsedRule, + existingLists, + }); + + importRuleResponse = [...importRuleResponse, ...exceptionErrors]; + + const importedRule = await detectionRulesClient.importRule({ + ruleToImport: { + ...parsedRule, + exceptions_list: [...exceptions], + }, + overwriteRules, + allowMissingConnectorSecrets, + }); + + resolve({ + rule_id: importedRule.rule_id, + status_code: 200, + }); + } catch (err) { + const { error, statusCode, message } = err; + resolve( + createBulkErrorObject({ + ruleId: parsedRule.rule_id, + statusCode: statusCode ?? error?.status_code ?? 400, + message: message ?? error?.message ?? 'unknown error', + }) + ); + } + } catch (error) { + reject(error); + } + }); + return [...accum, importsWorkerPromise]; + }, []) + ); + importRuleResponse = [...importRuleResponse, ...newImportRuleResponse]; + } + + return importRuleResponse; +}; From ba8d74aaa518e4db023aa678efd06bd86e7ebfe0 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 11 Sep 2024 13:46:44 -0500 Subject: [PATCH 055/120] Undo use of quotations This is a revert of a previous change to the copy, here. --- .../model/rule_schema/common_attributes.schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml index f1615a15fb3dc..088153cca4885 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml @@ -43,7 +43,7 @@ components: IsRuleImmutable: type: boolean deprecated: true - description: 'This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field.' + description: This field determines whether the rule is a prebuilt Elastic rule. It will be replaced with the `rule_source` field. IsExternalRuleCustomized: type: boolean From e49b4b2f33402093f9b1c015039990900d9e7102 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 11 Sep 2024 14:00:41 -0500 Subject: [PATCH 056/120] Revert mock changes While the version field is now optional again, I believe the addition of these fields is adding noise to the existing tests, which aren't expecting it. I'm going to instead first isolate the failing tests, and then add tests for explicitly specifying version(s). --- .../rule_management/import_rules/rule_to_import.mock.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts index 8d1a2bd8f859d..2b36645363edc 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts @@ -18,7 +18,6 @@ export const getImportRulesSchemaMock = (rewrites?: Partial): Rule language: 'kuery', rule_id: 'rule-1', immutable: false, - version: 1, ...rewrites, } as RuleToImport); @@ -32,7 +31,6 @@ export const getImportRulesWithIdSchemaMock = (ruleId = 'rule-1'): RuleToImport risk_score: 55, language: 'kuery', rule_id: ruleId, - version: 1, immutable: false, }); @@ -89,7 +87,6 @@ export const getImportThreatMatchRulesSchemaMock = ( }, ], immutable: false, - version: 1, ...rewrites, } as RuleToImport); From de28ac959d21ae590800291befd2e720d558c517 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 12 Sep 2024 15:13:37 -0500 Subject: [PATCH 057/120] Define our new "validated" RuleToImport Because we can't make the existing RuleToImport more strict (backwards compatibility), but we need `version` to be required for prebuilt rule customization, we now perform this check at runtime and continue to use RuleToImport for parsing/validation. Once the prebuilt rule epic is complete, we can replace RuleToImport with ValidatedRuleToImport, but for now, all our new code dealing with this new code path works with ValidatedRuleToImport. --- .../import_rules/rule_to_import.ts | 17 +++++++++++++++++ .../import_rules/rule_to_import_validation.ts | 5 ++++- .../rule_source/calculate_is_customized.ts | 7 ++----- .../calculate_rule_source_from_asset.ts | 4 ++-- .../import/calculate_rule_source_for_import.ts | 16 ++++++++++++---- .../logic/import/import_rules_with_source.ts | 7 +++++-- 6 files changed, 42 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index 56b3b957f8508..0d69848b50175 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -12,6 +12,7 @@ import { RequiredFieldInput, RuleSignatureId, TypeSpecificCreateProps, + RuleVersion, } from '../../model/rule_schema'; /** @@ -40,3 +41,19 @@ export const RuleToImport = BaseCreateProps.and(TypeSpecificCreateProps).and( required_fields: z.array(RequiredFieldInput).optional(), }) ); + +/** + * This type represents new rules being imported once the prebuilt rule + * customization work is complete. In order to provide backwards compatibility + * with existing rules, and not change behavior, we now validate `version` in + * the route as opposed to the type itself. + * + * It differs from RuleToImport in that it requires a `version` field. + */ +export type ValidatedRuleToImport = z.infer; +export type ValidatedRuleToImportInput = z.input; +export const ValidatedRuleToImport = RuleToImport.and( + z.object({ + version: RuleVersion, + }) +); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts index de21ac3a7964c..cef6d0fa03685 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import_validation.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { RuleToImport } from './rule_to_import'; +import type { RuleToImport, ValidatedRuleToImport } from './rule_to_import'; /** * Additional validation that is implemented outside of the schema itself. @@ -55,3 +55,6 @@ const validateThreshold = (rule: RuleToImport): string[] => { } return errors; }; + +export const ruleToImportHasVersion = (rule: RuleToImport): rule is ValidatedRuleToImport => + !!rule.version; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts index c7b7fdf2509b7..92c7d941dd7f1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts @@ -5,10 +5,7 @@ * 2.0. */ -import type { - RuleToImport, - RuleResponse, -} from '../../../../../../../../common/api/detection_engine'; +import type { RuleResponse } from '../../../../../../../../common/api/detection_engine'; import { MissingVersion } from '../../../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateRuleFieldsDiff } from '../../../../../prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff'; @@ -17,7 +14,7 @@ import { convertPrebuiltRuleAssetToRuleResponse } from '../../converters/convert export function calculateIsCustomized( baseRule: PrebuiltRuleAsset | undefined, - nextRule: RuleResponse | RuleToImport + nextRule: RuleResponse ) { if (baseRule == null) { // If the base version is missing, we consider the rule to be customized diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts index 285b476067c0d..a781d46e9d158 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { RuleSource, RuleToImport } from '../../../../../../../../common/api/detection_engine'; +import type { RuleResponse, RuleSource } from '../../../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateIsCustomized } from './calculate_is_customized'; @@ -25,7 +25,7 @@ export const calculateRuleSourceFromAsset = ({ assetWithMatchingVersion, ruleIdExists, }: { - rule: RuleToImport; + rule: RuleResponse; assetWithMatchingVersion: PrebuiltRuleAsset | undefined; ruleIdExists: boolean; }): RuleSource => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 67d0e4f0231d3..7a0a7cbfcb71c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -5,8 +5,12 @@ * 2.0. */ -import type { RuleToImport, RuleSource } from '../../../../../../common/api/detection_engine'; +import type { + RuleSource, + ValidatedRuleToImport, +} from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; +import { convertPrebuiltRuleAssetToRuleResponse } from '../detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response'; import { calculateRuleSourceFromAsset } from '../detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset'; /** @@ -24,7 +28,7 @@ export const calculateRuleSourceForImport = ({ prebuiltRuleAssets, installedRuleIds, }: { - rule: RuleToImport; + rule: ValidatedRuleToImport; prebuiltRuleAssets: PrebuiltRuleAsset[]; installedRuleIds: string[]; }): { ruleSource: RuleSource; immutable: boolean } => { @@ -32,9 +36,13 @@ export const calculateRuleSourceForImport = ({ (asset) => asset.rule_id === rule.rule_id ); const ruleIdExists = installedRuleIds.includes(rule.rule_id); - + // We convert here so that RuleSource calculation can + // continue to deal only with RuleResponses. The fields missing from the + // incoming rule are not actually needed for the calculation, but only to + // satisfy the type system. + const ruleResponseForImport = convertPrebuiltRuleAssetToRuleResponse(rule); const ruleSource = calculateRuleSourceFromAsset({ - rule, + rule: ruleResponseForImport, assetWithMatchingVersion, ruleIdExists, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts index 05470fbfd93a5..c50ef900c3c77 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts @@ -12,7 +12,10 @@ import type { ImportExceptionListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; +import { + ruleToImportHasVersion, + type RuleToImport, +} from '../../../../../../common/api/detection_engine/rule_management'; import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; @@ -100,7 +103,7 @@ export const importRules = async ({ return null; } - if (!parsedRule.version) { + if (!ruleToImportHasVersion(parsedRule)) { resolve( createBulkErrorObject({ statusCode: 400, From 23055086c640f2a59f6782ba3b2cdbf9b6330241 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 12 Sep 2024 15:51:56 -0500 Subject: [PATCH 058/120] Add new methods to our test mocks --- .../detection_rules_client/__mocks__/detection_rules_client.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts index b6d14a307801e..13299af6f4201 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts @@ -17,7 +17,9 @@ const createDetectionRulesClientMock = () => { patchRule: jest.fn(), deleteRule: jest.fn(), upgradePrebuiltRule: jest.fn(), + legacyImportRule: jest.fn(), importRule: jest.fn(), + importRules: jest.fn(), }; return mocked; }; From d5091dcf9ae8395cdd15ff92a91a61fa0455fc07 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 12 Sep 2024 21:13:00 -0500 Subject: [PATCH 059/120] Add missing test We were effectively testing 3/4 of the permutations of this function. --- .../calculate_rule_source_for_import.test.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts index b35ddb4142429..c982a9b790b3b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -63,4 +63,24 @@ describe('calculateRuleSourceForImport', () => { immutable: true, }); }); + + it('calculates as external without customizations if an exact match is found', () => { + const rule = getRulesSchemaMock(); + rule.rule_id = 'rule_id'; + const prebuiltRuleAssets = [getPrebuiltRuleMock(rule)]; + + const result = calculateRuleSourceForImport({ + rule, + prebuiltRuleAssets, + installedRuleIds: ['rule_id'], + }); + + expect(result).toEqual({ + ruleSource: { + type: 'external', + is_customized: false, + }, + immutable: true, + }); + }); }); From 47350da4ab5f346885bfca6f6dbda0413e21f4f3 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 14:38:17 -0500 Subject: [PATCH 060/120] More wrangling of types * Makes old functions/arguments/types more explicit with a "Legacy" prefix * Addresses backwards-compatibility issue caught by typescript: Adds back the defaulting of immutability to "false" in the old import code --- .../detection_rules_client.ts | 3 ++- .../detection_rules_client_interface.ts | 14 +++++++++++--- .../detection_rules_client/methods/import_rule.ts | 12 +++++++----- .../methods/import_rule_with_source.ts | 11 ++--------- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts index a1d65e5bbf6ce..d7dc069d2b6cd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts @@ -22,6 +22,7 @@ import type { IDetectionRulesClient, ImportRuleArgs, ImportRulesArgs, + LegacyImportRuleArgs, PatchRuleArgs, UpdateRuleArgs, UpgradePrebuiltRuleArgs, @@ -124,7 +125,7 @@ export const createDetectionRulesClient = ({ }); }, - async legacyImportRule(args: ImportRuleArgs): Promise { + async legacyImportRule(args: LegacyImportRuleArgs): Promise { return withSecuritySpan('DetectionRulesClient.legacyImportRule', async () => { return importRule({ actionsClient, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts index c3b2bf4862f8e..16dbbd98d9b42 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts @@ -11,8 +11,10 @@ import type { RuleUpdateProps, RulePatchProps, RuleObjectId, - RuleToImport, RuleResponse, + ValidatedRuleToImport, + RuleToImport, + RuleSource, } from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; @@ -26,7 +28,7 @@ export interface IDetectionRulesClient { patchRule: (args: PatchRuleArgs) => Promise; deleteRule: (args: DeleteRuleArgs) => Promise; upgradePrebuiltRule: (args: UpgradePrebuiltRuleArgs) => Promise; - legacyImportRule: (args: ImportRuleArgs) => Promise; + legacyImportRule: (args: LegacyImportRuleArgs) => Promise; importRule: (args: ImportRuleArgs) => Promise; importRules: (args: ImportRulesArgs) => Promise; } @@ -55,12 +57,18 @@ export interface UpgradePrebuiltRuleArgs { ruleAsset: PrebuiltRuleAsset; } -export interface ImportRuleArgs { +export interface LegacyImportRuleArgs { ruleToImport: RuleToImport; overwriteRules?: boolean; allowMissingConnectorSecrets?: boolean; } +export interface ImportRuleArgs { + ruleToImport: ValidatedRuleToImport & { rule_source: RuleSource; immutable: boolean }; + overwriteRules?: boolean; + allowMissingConnectorSecrets?: boolean; +} + export interface ImportRulesArgs { ruleChunks: PromiseFromStreams[][]; rulesResponseAcc: ImportRuleResponse[]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts index 4066cb00849a3..c60f7c0098655 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts @@ -14,7 +14,7 @@ import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic import { createBulkErrorObject } from '../../../../routes/utils'; import { convertAlertingRuleToRuleResponse } from '../converters/convert_alerting_rule_to_rule_response'; import { convertRuleResponseToAlertingRule } from '../converters/convert_rule_response_to_alerting_rule'; -import type { ImportRuleArgs } from '../detection_rules_client_interface'; +import type { LegacyImportRuleArgs } from '../detection_rules_client_interface'; import { applyRuleUpdate } from '../mergers/apply_rule_update'; import { validateMlAuth, toggleRuleEnabledOnUpdate } from '../utils'; import { createRule } from './create_rule'; @@ -24,7 +24,7 @@ interface ImportRuleOptions { actionsClient: ActionsClient; rulesClient: RulesClient; prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; - importRulePayload: ImportRuleArgs; + importRulePayload: LegacyImportRuleArgs; mlAuthz: MlAuthz; } @@ -36,12 +36,14 @@ export const importRule = async ({ mlAuthz, }: ImportRuleOptions): Promise => { const { ruleToImport, overwriteRules, allowMissingConnectorSecrets } = importRulePayload; + // For backwards compatibility in this legacy path, immutable is always false. + const rule = { ...ruleToImport, immutable: false }; await validateMlAuth(mlAuthz, ruleToImport.type); const existingRule = await getRuleByRuleId({ rulesClient, - ruleId: ruleToImport.rule_id, + ruleId: rule.rule_id, }); if (existingRule && !overwriteRules) { @@ -56,7 +58,7 @@ export const importRule = async ({ const ruleWithUpdates = await applyRuleUpdate({ prebuiltRuleAssetClient, existingRule, - ruleUpdate: ruleToImport, + ruleUpdate: rule, }); const updatedRule = await rulesClient.update({ @@ -75,7 +77,7 @@ export const importRule = async ({ actionsClient, rulesClient, mlAuthz, - rule: ruleToImport, + rule, allowMissingConnectorSecrets, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts index 29f64e7cc2189..454b9ad6cc81c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts @@ -9,10 +9,7 @@ import type { RulesClient } from '@kbn/alerting-plugin/server'; import type { ActionsClient } from '@kbn/actions-plugin/server'; import { ruleTypeMappings } from '@kbn/securitysolution-rules'; -import type { - RuleResponse, - RuleSource, -} from '../../../../../../../common/api/detection_engine/model/rule_schema'; +import type { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; import type { MlAuthz } from '../../../../../machine_learning/authz'; import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import { createBulkErrorObject } from '../../../../routes/utils'; @@ -25,15 +22,11 @@ import { validateMlAuth } from '../utils'; import { getRuleByRuleId } from './get_rule_by_rule_id'; import { SERVER_APP_ID } from '../../../../../../../common'; -interface ImportRuleWithSourceArgs extends ImportRuleArgs { - ruleToImport: ImportRuleArgs['ruleToImport'] & { rule_source: RuleSource; immutable: boolean }; -} - interface ImportRuleOptions { actionsClient: ActionsClient; rulesClient: RulesClient; prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; - importRulePayload: ImportRuleWithSourceArgs; + importRulePayload: ImportRuleArgs; mlAuthz: MlAuthz; } From 538620cf4132157fc1dcdde5bdc50c6904efce90 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 14:49:05 -0500 Subject: [PATCH 061/120] Define DiffableRuleInput as type used for calculating rule_source For the purposes of calculating rule_source (and mainly for diffing to calculate `is_customized`), we now have multiple rule representations being compared, where before all comparisons were on the much broader `RuleResponse`, and different rule representations were converted to `RuleResponse` before being compared. In reality, a good number of RuleResponse fields are not used for comparison, and as is evidenced by this commit, what's needed is (at most) `RuleToImport` + `version` (represented now by `ValidatedRuleToImport`). It's confusing, and unnecessary, to convert this type to a `RuleResponse` for the purposes of comparison. Luckily, a type system can do much of this work for us. Ideally, the comparison code would accept the minimal representation that it needs, and our extraction/comparison utilities would work on that. For now, though, in order to minimize the number of changes needed, I've instead defined the type, `DiffableRuleInput`, to be equivalent to the more minimal input, `ValidatedRuleToImport`. Since `RuleResponse` is a superset of that type, no existing code changes are needed. If/when we need, we can define exactly the fields needed for comparison as a standalone type, but for now this is a step in the right direction. --- .../diff/convert_rule_to_diffable.ts | 6 +++--- .../diff/extract_building_block_object.ts | 6 ++++-- .../diff/extract_rule_name_override_object.ts | 4 ++-- .../prebuilt_rules/diff/extract_rule_schedule.ts | 5 +++-- .../diff/extract_timeline_template_reference.ts | 4 ++-- .../diff/extract_timestamp_override_object.ts | 4 ++-- .../detection_engine/prebuilt_rules/diff/types.ts | 14 ++++++++++++++ .../mergers/rule_source/calculate_is_customized.ts | 4 ++-- .../calculate_rule_source_from_asset.ts | 5 +++-- .../import/calculate_rule_source_for_import.ts | 8 +------- 10 files changed, 36 insertions(+), 24 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/types.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts index 45b4612e83c8e..77b8011e87b87 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts @@ -20,7 +20,6 @@ import type { NewTermsRuleCreateProps, QueryRule, QueryRuleCreateProps, - RuleResponse, SavedQueryRule, SavedQueryRuleCreateProps, ThreatMatchRule, @@ -41,6 +40,7 @@ import type { DiffableThresholdFields, } from '../../../api/detection_engine/prebuilt_rules'; import { addEcsToRequiredFields } from '../../rule_management/utils'; +import type { DiffableRuleInput } from './types'; import { extractBuildingBlockObject } from './extract_building_block_object'; import { extractInlineKqlQuery, @@ -58,7 +58,7 @@ import { extractTimestampOverrideObject } from './extract_timestamp_override_obj * Normalizes a given rule to the form which is suitable for passing to the diff algorithm. * Read more in the JSDoc description of DiffableRule. */ -export const convertRuleToDiffable = (rule: RuleResponse): DiffableRule => { +export const convertRuleToDiffable = (rule: DiffableRuleInput): DiffableRule => { const commonFields = extractDiffableCommonFields(rule); switch (rule.type) { @@ -108,7 +108,7 @@ export const convertRuleToDiffable = (rule: RuleResponse): DiffableRule => { }; const extractDiffableCommonFields = ( - rule: RuleResponse + rule: DiffableRuleInput ): RequiredOptional => { return { // --------------------- REQUIRED FIELDS diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_building_block_object.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_building_block_object.ts index 18d6ebfb54ce4..eae1350471701 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_building_block_object.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_building_block_object.ts @@ -5,10 +5,12 @@ * 2.0. */ -import type { RuleResponse } from '../../../api/detection_engine/model/rule_schema'; import type { BuildingBlockObject } from '../../../api/detection_engine/prebuilt_rules'; +import type { DiffableRuleInput } from './types'; -export const extractBuildingBlockObject = (rule: RuleResponse): BuildingBlockObject | undefined => { +export const extractBuildingBlockObject = ( + rule: DiffableRuleInput +): BuildingBlockObject | undefined => { if (rule.building_block_type == null) { return undefined; } diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_name_override_object.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_name_override_object.ts index 55119608264f9..13eb32f4c794f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_name_override_object.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_name_override_object.ts @@ -5,11 +5,11 @@ * 2.0. */ -import type { RuleResponse } from '../../../api/detection_engine/model/rule_schema'; import type { RuleNameOverrideObject } from '../../../api/detection_engine/prebuilt_rules'; +import type { DiffableRuleInput } from './types'; export const extractRuleNameOverrideObject = ( - rule: RuleResponse + rule: DiffableRuleInput ): RuleNameOverrideObject | undefined => { if (rule.rule_name_override == null) { return undefined; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_schedule.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_schedule.ts index 6812a0a2f6fc6..5ebf1e96c0769 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_schedule.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_schedule.ts @@ -9,10 +9,11 @@ import moment from 'moment'; import dateMath from '@elastic/datemath'; import { parseDuration } from '@kbn/alerting-plugin/common'; -import type { RuleMetadata, RuleResponse } from '../../../api/detection_engine/model/rule_schema'; +import type { RuleMetadata } from '../../../api/detection_engine/model/rule_schema'; import type { RuleSchedule } from '../../../api/detection_engine/prebuilt_rules'; +import type { DiffableRuleInput } from './types'; -export const extractRuleSchedule = (rule: RuleResponse): RuleSchedule => { +export const extractRuleSchedule = (rule: DiffableRuleInput): RuleSchedule => { const interval = rule.interval ?? '5m'; const from = rule.from ?? 'now-6m'; const to = rule.to ?? 'now'; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timeline_template_reference.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timeline_template_reference.ts index 7dcce30061659..d347144dd4da3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timeline_template_reference.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timeline_template_reference.ts @@ -5,11 +5,11 @@ * 2.0. */ -import type { RuleResponse } from '../../../api/detection_engine/model/rule_schema'; import type { TimelineTemplateReference } from '../../../api/detection_engine/prebuilt_rules'; +import type { DiffableRuleInput } from './types'; export const extractTimelineTemplateReference = ( - rule: RuleResponse + rule: DiffableRuleInput ): TimelineTemplateReference | undefined => { if (rule.timeline_id == null) { return undefined; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timestamp_override_object.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timestamp_override_object.ts index 40f218fe430b7..588df49b9b969 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timestamp_override_object.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timestamp_override_object.ts @@ -5,11 +5,11 @@ * 2.0. */ -import type { RuleResponse } from '../../../api/detection_engine/model/rule_schema'; import type { TimestampOverrideObject } from '../../../api/detection_engine/prebuilt_rules'; +import type { DiffableRuleInput } from './types'; export const extractTimestampOverrideObject = ( - rule: RuleResponse + rule: DiffableRuleInput ): TimestampOverrideObject | undefined => { if (rule.timestamp_override == null) { return undefined; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/types.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/types.ts new file mode 100644 index 0000000000000..46ed266a941be --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/types.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ValidatedRuleToImport } from '../../../api/detection_engine'; + +/** + * This type represents the minimal rule representation that can be + * converted to the DiffableRule type. + */ +export type DiffableRuleInput = ValidatedRuleToImport; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts index 92c7d941dd7f1..88b47063e78df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { RuleResponse } from '../../../../../../../../common/api/detection_engine'; +import type { DiffableRuleInput } from '../../../../../../../../common/detection_engine/prebuilt_rules/diff/types'; import { MissingVersion } from '../../../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateRuleFieldsDiff } from '../../../../../prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff'; @@ -14,7 +14,7 @@ import { convertPrebuiltRuleAssetToRuleResponse } from '../../converters/convert export function calculateIsCustomized( baseRule: PrebuiltRuleAsset | undefined, - nextRule: RuleResponse + nextRule: DiffableRuleInput ) { if (baseRule == null) { // If the base version is missing, we consider the rule to be customized diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts index a781d46e9d158..7766b286f8e2e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts @@ -5,7 +5,8 @@ * 2.0. */ -import type { RuleResponse, RuleSource } from '../../../../../../../../common/api/detection_engine'; +import type { RuleSource } from '../../../../../../../../common/api/detection_engine'; +import type { DiffableRuleInput } from '../../../../../../../../common/detection_engine/prebuilt_rules/diff/types'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateIsCustomized } from './calculate_is_customized'; @@ -25,7 +26,7 @@ export const calculateRuleSourceFromAsset = ({ assetWithMatchingVersion, ruleIdExists, }: { - rule: RuleResponse; + rule: DiffableRuleInput; assetWithMatchingVersion: PrebuiltRuleAsset | undefined; ruleIdExists: boolean; }): RuleSource => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 7a0a7cbfcb71c..d2146bf047d8c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -10,7 +10,6 @@ import type { ValidatedRuleToImport, } from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; -import { convertPrebuiltRuleAssetToRuleResponse } from '../detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response'; import { calculateRuleSourceFromAsset } from '../detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset'; /** @@ -36,13 +35,8 @@ export const calculateRuleSourceForImport = ({ (asset) => asset.rule_id === rule.rule_id ); const ruleIdExists = installedRuleIds.includes(rule.rule_id); - // We convert here so that RuleSource calculation can - // continue to deal only with RuleResponses. The fields missing from the - // incoming rule are not actually needed for the calculation, but only to - // satisfy the type system. - const ruleResponseForImport = convertPrebuiltRuleAssetToRuleResponse(rule); const ruleSource = calculateRuleSourceFromAsset({ - rule: ruleResponseForImport, + rule, assetWithMatchingVersion, ruleIdExists, }); From 2d56f12cdd8ead5c78e2bfadd4dc2a852e5cf3ab Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 16:52:02 -0500 Subject: [PATCH 062/120] Fixing existing tests and types This adds a new representation for the post-calculation version of an importing rule: `ValidatedRuleToImportWithSource`. This also fixes some existing tests that were broken due to the wrong inputs being used. --- .../import_rules/rule_to_import.mock.ts | 23 +- .../import_rules/rule_to_import.test.ts | 11 + .../import_rules/rule_to_import.ts | 16 ++ .../api/rules/import_rules/route.test.ts | 1 - ...detection_rules_client.import_rule.test.ts | 18 +- ...on_rules_client.legacy_import_rule.test.ts | 247 ++++++++++++++++++ .../methods/import_rule_with_source.test.ts | 20 +- 7 files changed, 315 insertions(+), 21 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.legacy_import_rule.test.ts diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts index 2b36645363edc..94824a0aac2cd 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts @@ -5,7 +5,11 @@ * 2.0. */ -import type { RuleToImport } from './rule_to_import'; +import type { + RuleToImport, + ValidatedRuleToImport, + ValidatedRuleToImportWithSource, +} from './rule_to_import'; export const getImportRulesSchemaMock = (rewrites?: Partial): RuleToImport => ({ @@ -21,6 +25,23 @@ export const getImportRulesSchemaMock = (rewrites?: Partial): Rule ...rewrites, } as RuleToImport); +export const getValidatedRuleToImportMock = ( + overrides?: Partial +): ValidatedRuleToImport => ({ + version: 1, + ...getImportRulesSchemaMock(overrides), +}); + +export const getValidatedRuleToImportWithSourceMock = ( + overrides?: Partial +): ValidatedRuleToImportWithSource => ({ + rule_source: { + type: 'internal', + }, + immutable: false, + ...getValidatedRuleToImportMock(overrides), +}); + export const getImportRulesWithIdSchemaMock = (ruleId = 'rule-1'): RuleToImport => ({ id: '6afb8ce1-ea94-4790-8653-fd0b021d2113', description: 'some description', diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts index 8e7415b7c2729..e3bd7992398a4 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts @@ -1087,5 +1087,16 @@ describe('RuleToImport', () => { expectParseSuccess(result); expect(result.data).toEqual(payload); }); + + describe('backwards compatibility', () => { + it('allows version to be absent', () => { + const payload = getImportRulesSchemaMock(); + delete payload.version; + + const result = RuleToImport.safeParse(payload); + expectParseSuccess(result); + expect(result.data).toEqual(payload); + }); + }); }); }); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index 0d69848b50175..e97c9abf47259 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -13,6 +13,7 @@ import { RuleSignatureId, TypeSpecificCreateProps, RuleVersion, + RuleSource, } from '../../model/rule_schema'; /** @@ -57,3 +58,18 @@ export const ValidatedRuleToImport = RuleToImport.and( version: RuleVersion, }) ); + +/** + * This type represents new rules being imported once the prebuilt rule + * customization work is complete. In addition to having been validated to have + * `version`, they've also had their `rule_source` and `immutable` fields + * calculated, and are ready to be persisted. + */ +export type ValidatedRuleToImportWithSource = z.infer; +export type ValidatedRuleToImportWithSourceInput = z.input; +export const ValidatedRuleToImportWithSource = ValidatedRuleToImport.and( + z.object({ + rule_source: RuleSource, + immutable: z.boolean(), + }) +); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts index e0cf057f0d39a..e3588b5beae32 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts @@ -275,7 +275,6 @@ describe('Import rules route', () => { it('returns an error if rule is missing a version', async () => { const ruleWithoutVersion = getImportRulesWithIdSchemaMock('rule-1'); - // @ts-expect-error delete ruleWithoutVersion.version; const payload = buildHapiStream(rulesToNdJsonString([ruleWithoutVersion])); const versionlessRequest = getImportRulesRequest(payload); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts index e3b922fa831a6..7b64d6daa76de 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts @@ -9,10 +9,7 @@ import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; import type { ActionsClient } from '@kbn/actions-plugin/server'; import { savedObjectsClientMock } from '@kbn/core/server/mocks'; -import { - getCreateRulesSchemaMock, - getRulesSchemaMock, -} from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; +import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; import { buildMlAuthz } from '../../../../machine_learning/authz'; import { throwAuthzError } from '../../../../machine_learning/validation'; import { getRuleMock } from '../../../routes/__mocks__/request_responses'; @@ -20,6 +17,7 @@ import { getQueryRuleParams } from '../../../rule_schema/mocks'; import { createDetectionRulesClient } from './detection_rules_client'; import type { IDetectionRulesClient } from './detection_rules_client_interface'; import { getRuleByRuleId } from './methods/get_rule_by_rule_id'; +import { getValidatedRuleToImportWithSourceMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; jest.mock('../../../../machine_learning/authz'); jest.mock('../../../../machine_learning/validation'); @@ -33,14 +31,12 @@ describe('DetectionRulesClient.importRule', () => { const mlAuthz = (buildMlAuthz as jest.Mock)(); let actionsClient: jest.Mocked; - const immutable = false as const; // Can only take value of false const allowMissingConnectorSecrets = true; const ruleToImport = { - ...getCreateRulesSchemaMock(), + ...getValidatedRuleToImportWithSourceMock(), tags: ['import-tag'], rule_id: 'rule-id', version: 1, - immutable, }; const existingRule = getRulesSchemaMock(); existingRule.rule_id = ruleToImport.rule_id; @@ -72,9 +68,10 @@ describe('DetectionRulesClient.importRule', () => { name: ruleToImport.name, tags: ruleToImport.tags, params: expect.objectContaining({ - immutable, + immutable: ruleToImport.immutable, ruleId: ruleToImport.rule_id, version: ruleToImport.version, + ruleSource: ruleToImport.rule_source, }), }), allowMissingConnectorSecrets, @@ -115,8 +112,11 @@ describe('DetectionRulesClient.importRule', () => { name: ruleToImport.name, tags: ruleToImport.tags, params: expect.objectContaining({ - index: ruleToImport.index, description: ruleToImport.description, + immutable: ruleToImport.immutable, + ruleId: ruleToImport.rule_id, + version: ruleToImport.version, + ruleSource: ruleToImport.rule_source, }), }), id: existingRule.id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.legacy_import_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.legacy_import_rule.test.ts new file mode 100644 index 0000000000000..5108729cc9cc8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.legacy_import_rule.test.ts @@ -0,0 +1,247 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; + +import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { + getCreateRulesSchemaMock, + getRulesSchemaMock, +} from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; +import { buildMlAuthz } from '../../../../machine_learning/authz'; +import { throwAuthzError } from '../../../../machine_learning/validation'; +import { getRuleMock } from '../../../routes/__mocks__/request_responses'; +import { getQueryRuleParams } from '../../../rule_schema/mocks'; +import { createDetectionRulesClient } from './detection_rules_client'; +import type { IDetectionRulesClient } from './detection_rules_client_interface'; +import { getRuleByRuleId } from './methods/get_rule_by_rule_id'; + +jest.mock('../../../../machine_learning/authz'); +jest.mock('../../../../machine_learning/validation'); + +jest.mock('./methods/get_rule_by_rule_id'); + +describe('DetectionRulesClient.legacyImportRule', () => { + let rulesClient: ReturnType; + let detectionRulesClient: IDetectionRulesClient; + + const mlAuthz = (buildMlAuthz as jest.Mock)(); + let actionsClient: jest.Mocked; + + const immutable = false as const; // Can only take value of false + const allowMissingConnectorSecrets = true; + const ruleToImport = { + ...getCreateRulesSchemaMock(), + tags: ['import-tag'], + rule_id: 'rule-id', + version: 1, + immutable, + }; + const existingRule = getRulesSchemaMock(); + existingRule.rule_id = ruleToImport.rule_id; + + beforeEach(() => { + rulesClient = rulesClientMock.create(); + rulesClient.create.mockResolvedValue(getRuleMock(getQueryRuleParams())); + rulesClient.update.mockResolvedValue(getRuleMock(getQueryRuleParams())); + const savedObjectsClient = savedObjectsClientMock.create(); + detectionRulesClient = createDetectionRulesClient({ + actionsClient, + rulesClient, + mlAuthz, + savedObjectsClient, + }); + }); + + it('calls rulesClient.create with the correct parameters when rule_id does not match an installed rule', async () => { + (getRuleByRuleId as jest.Mock).mockResolvedValueOnce(null); + await detectionRulesClient.legacyImportRule({ + ruleToImport, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + name: ruleToImport.name, + tags: ruleToImport.tags, + params: expect.objectContaining({ + immutable, + ruleId: ruleToImport.rule_id, + version: ruleToImport.version, + }), + }), + allowMissingConnectorSecrets, + }) + ); + }); + + it('throws if mlAuth fails', async () => { + (throwAuthzError as jest.Mock).mockImplementationOnce(() => { + throw new Error('mocked MLAuth error'); + }); + + await expect( + detectionRulesClient.legacyImportRule({ + ruleToImport, + overwriteRules: true, + allowMissingConnectorSecrets, + }) + ).rejects.toThrow('mocked MLAuth error'); + + expect(rulesClient.create).not.toHaveBeenCalled(); + expect(rulesClient.update).not.toHaveBeenCalled(); + }); + + describe('when rule_id matches an installed rule', () => { + it('calls rulesClient.update with the correct parameters when overwriteRules is true', async () => { + (getRuleByRuleId as jest.Mock).mockResolvedValueOnce(existingRule); + + await detectionRulesClient.legacyImportRule({ + ruleToImport, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + name: ruleToImport.name, + tags: ruleToImport.tags, + params: expect.objectContaining({ + index: ruleToImport.index, + description: ruleToImport.description, + }), + }), + id: existingRule.id, + }) + ); + }); + + /** + * Existing rule may have nullable fields set to a value (e.g. `timestamp_override` is set to `some.value`) but + * a rule to import doesn't have these fields set (e.g. `timestamp_override` is NOT present at all in the ndjson file). + * We expect the updated rule won't have such fields preserved (e.g. `timestamp_override` will be removed). + * + * Unit test is only able to check `updateRules()` receives a proper update object. + */ + it('ensures overwritten rule DOES NOT preserve fields missed in the imported rule when "overwriteRules" is "true" and matching rule found', async () => { + const existingRuleWithTimestampOverride = { + ...existingRule, + timestamp_override: '2020-01-01T00:00:00Z', + }; + (getRuleByRuleId as jest.Mock).mockResolvedValue(existingRuleWithTimestampOverride); + + await detectionRulesClient.legacyImportRule({ + ruleToImport: { + ...ruleToImport, + timestamp_override: undefined, + }, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).not.toHaveBeenCalled(); + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + id: existingRule.id, + data: expect.not.objectContaining({ + timestamp_override: expect.anything(), + timestampOverride: expect.anything(), + }), + }) + ); + }); + + it('enables the rule if the imported rule has enabled: true', async () => { + const disabledExistingRule = { + ...existingRule, + enabled: false, + }; + (getRuleByRuleId as jest.Mock).mockResolvedValueOnce(disabledExistingRule); + + const rule = await detectionRulesClient.legacyImportRule({ + ruleToImport: { + ...ruleToImport, + enabled: true, + }, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).not.toHaveBeenCalled(); + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + id: existingRule.id, + data: expect.not.objectContaining({ + enabled: expect.anything(), + }), + }) + ); + + expect(rule.enabled).toBe(true); + expect(rulesClient.enableRule).toHaveBeenCalledWith( + expect.objectContaining({ + id: existingRule.id, + }) + ); + }); + + it('disables the rule if the imported rule has enabled: false', async () => { + const enabledExistingRule = { + ...existingRule, + enabled: true, + }; + (getRuleByRuleId as jest.Mock).mockResolvedValueOnce(enabledExistingRule); + + const rule = await detectionRulesClient.legacyImportRule({ + ruleToImport: { + ...ruleToImport, + enabled: false, + }, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).not.toHaveBeenCalled(); + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + id: existingRule.id, + data: expect.not.objectContaining({ + enabled: expect.anything(), + }), + }) + ); + + expect(rule.enabled).toBe(false); + expect(rulesClient.disableRule).toHaveBeenCalledWith( + expect.objectContaining({ + id: existingRule.id, + }) + ); + }); + + it('rejects when overwriteRules is false', async () => { + (getRuleByRuleId as jest.Mock).mockResolvedValue(existingRule); + await expect( + detectionRulesClient.legacyImportRule({ + ruleToImport, + overwriteRules: false, + allowMissingConnectorSecrets, + }) + ).rejects.toMatchObject({ + error: { + status_code: 409, + message: `rule_id: "${ruleToImport.rule_id}" already exists`, + }, + rule_id: ruleToImport.rule_id, + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts index 13c507d1570a9..b27133ea74b0d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts @@ -8,7 +8,7 @@ import { actionsClientMock } from '@kbn/actions-plugin/server/actions_client/actions_client.mock'; import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; -import { getImportRulesSchemaMock } from '../../../../../../../common/api/detection_engine/rule_management/mocks'; +import { getValidatedRuleToImportMock } from '../../../../../../../common/api/detection_engine/rule_management/mocks'; import { importRuleWithSource } from './import_rule_with_source'; import { buildMlAuthz } from '../../../../../machine_learning/authz'; import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; @@ -55,7 +55,7 @@ describe('importRuleWithSource', () => { it('throws an error if overwriting is disabled', async () => { const rule = { - ...getImportRulesSchemaMock(), + ...getValidatedRuleToImportMock(), immutable: true, rule_source: { type: 'external' as const, @@ -81,7 +81,7 @@ describe('importRuleWithSource', () => { it('preserves the passed "rule_source" and "immutable" values', async () => { const rule = { - ...getImportRulesSchemaMock(), + ...getValidatedRuleToImportMock(), immutable: true, rule_source: { type: 'external' as const, @@ -114,7 +114,7 @@ describe('importRuleWithSource', () => { it('TODO REVIEW does not preserve the passed "enabled" value', async () => { const disabledRule = { - ...getImportRulesSchemaMock(), + ...getValidatedRuleToImportMock(), enabled: false, immutable: true, rule_source: { @@ -142,7 +142,7 @@ describe('importRuleWithSource', () => { it('TODO REVIEW preserves the existing rule\'s "id" value', async () => { const rule = { - ...getImportRulesSchemaMock(), + ...getValidatedRuleToImportMock(), immutable: true, id: 'some-id', rule_source: { @@ -173,7 +173,7 @@ describe('importRuleWithSource', () => { it('TODO REVIEW preserves the existing rule\'s "version" value if left unspecified', async () => { const rule = { - ...getImportRulesSchemaMock(), + ...getValidatedRuleToImportMock(), immutable: true, rule_source: { type: 'external' as const, @@ -202,7 +202,7 @@ describe('importRuleWithSource', () => { it('TODO REVIEW preserves the specified "version" value', async () => { const rule = { - ...getImportRulesSchemaMock(), + ...getValidatedRuleToImportMock(), immutable: true, version: 42, rule_source: { @@ -238,7 +238,7 @@ describe('importRuleWithSource', () => { it('preserves the passed "rule_source" and "immutable" values', async () => { const rule = { - ...getImportRulesSchemaMock(), + ...getValidatedRuleToImportMock(), immutable: true, rule_source: { type: 'external' as const, @@ -270,7 +270,7 @@ describe('importRuleWithSource', () => { it('preserves the passed "enabled" value', async () => { const rule = { - ...getImportRulesSchemaMock(), + ...getValidatedRuleToImportMock(), enabled: true, immutable: true, rule_source: { @@ -298,7 +298,7 @@ describe('importRuleWithSource', () => { it('defaults defaultable values', async () => { const rule = { - ...getImportRulesSchemaMock(), + ...getValidatedRuleToImportMock(), immutable: true, rule_source: { type: 'external' as const, From 9ffed8751b517dd3f2cda9394d2880051b8e34de Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 16:52:27 -0500 Subject: [PATCH 063/120] Update comment since this bug was already caught Nice work, Davis! --- .../detection_rules_client/mergers/apply_rule_update.ts | 2 +- .../methods/import_rule_with_source.ts | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts index 244431c0d9d48..b911e66a1fc45 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_update.ts @@ -28,7 +28,7 @@ export const applyRuleUpdate = async ({ ...applyRuleDefaults(ruleUpdate), // Use existing values - enabled: ruleUpdate.enabled ?? existingRule.enabled, // TODO this value isn't used by our call to update + enabled: ruleUpdate.enabled ?? existingRule.enabled, version: ruleUpdate.version ?? existingRule.version, // Always keep existing values for these fields diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts index 454b9ad6cc81c..9cb1a115c3471 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts @@ -18,7 +18,7 @@ import { convertRuleResponseToAlertingRule } from '../converters/convert_rule_re import type { ImportRuleArgs } from '../detection_rules_client_interface'; import { applyRuleUpdate } from '../mergers/apply_rule_update'; import { applyRuleDefaults } from '../mergers/apply_rule_defaults'; -import { validateMlAuth } from '../utils'; +import { validateMlAuth, toggleRuleEnabledOnUpdate } from '../utils'; import { getRuleByRuleId } from './get_rule_by_rule_id'; import { SERVER_APP_ID } from '../../../../../../../common'; @@ -74,7 +74,11 @@ export const importRuleWithSource = async ({ id: existingRule.id, data: convertRuleResponseToAlertingRule(ruleWithUpdates, actionsClient), }); - return convertAlertingRuleToRuleResponse(updatedRule); + + // We strip `enabled` from the rule object to use in the rules client and need to enable it separately if user has enabled the updated rule + const { enabled } = await toggleRuleEnabledOnUpdate(rulesClient, existingRule, ruleWithUpdates); + + return convertAlertingRuleToRuleResponse({ ...updatedRule, enabled }); } const ruleWithDefaults = applyRuleDefaults(ruleToImport); From ee66654d7d74d2eee3b9db0162c92314a9913b8d Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 16:52:52 -0500 Subject: [PATCH 064/120] Satisfy types by manually setting properties in new object argument Setting a definite type to an existing, typed object does not narrow the type in TS. --- .../rule_management/logic/import/import_rules_with_source.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts index c50ef900c3c77..dce10b105be89 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts @@ -127,8 +127,6 @@ export const importRules = async ({ prebuiltRuleAssets, installedRuleIds, }); - parsedRule.rule_source = ruleSource; - parsedRule.immutable = immutable; try { const [exceptionErrors, exceptions] = checkRuleExceptionReferences({ @@ -141,6 +139,8 @@ export const importRules = async ({ const importedRule = await detectionRulesClient.importRule({ ruleToImport: { ...parsedRule, + rule_source: ruleSource, + immutable, exceptions_list: [...exceptions], }, overwriteRules, From 7b0b79478bc817ac818635c70b53be58cef33fa4 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 17:09:23 -0500 Subject: [PATCH 065/120] Revert making rule_source required in RuleResponse While this is technically true, the current effort does not change this fact, and this change has significant impact on downstream types so would overinflate this branch. See [this discussion](https://github.com/elastic/kibana/pull/190198/files#r1734670790) for more context. --- .../api/detection_engine/model/rule_schema/rule_schemas.gen.ts | 2 +- .../detection_engine/model/rule_schema/rule_schemas.schema.yaml | 1 - ...urity_solution_detections_api_2023_10_31.bundled.schema.yaml | 1 - ...urity_solution_detections_api_2023_10_31.bundled.schema.yaml | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index 689eeb9aa10f5..2d3dbcd3f436f 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -160,7 +160,7 @@ export const ResponseFields = z.object({ id: RuleObjectId, rule_id: RuleSignatureId, immutable: IsRuleImmutable, - rule_source: RuleSource, + rule_source: RuleSource.optional(), updated_at: z.string().datetime(), updated_by: z.string(), created_at: z.string().datetime(), diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml index a6b82568df16d..4ade72c15fbb9 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml @@ -203,7 +203,6 @@ components: - id - rule_id - immutable - - rule_source - updated_at - updated_by - created_at diff --git a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml index b854dc2bc4475..dcee1694a4aeb 100644 --- a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -4869,7 +4869,6 @@ components: - id - rule_id - immutable - - rule_source - updated_at - updated_by - created_at diff --git a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml index 3e182c1112d5d..e3a294c9f92a5 100644 --- a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -4022,7 +4022,6 @@ components: - id - rule_id - immutable - - rule_source - updated_at - updated_by - created_at From 2568e5cb53b252e8dfb97ad23e68ca20e111d797 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 17:13:00 -0500 Subject: [PATCH 066/120] Revert "Update mocks and tests to reflect required param rule_source" This reverts commit 1c5834b71d4c79f0234168f2f0ef8a4df092392e. --- .../model/rule_schema/rule_response_schema.mock.ts | 1 - .../model/rule_schema/rule_response_schema.test.ts | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.mock.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.mock.ts index 59b09533a6e14..c605436576995 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.mock.ts @@ -47,7 +47,6 @@ const getResponseBaseParams = (anchorDate: string = ANCHOR_DATE): SharedResponse risk_score: 55, risk_score_mapping: [], rule_id: 'query-rule-id', - rule_source: { type: 'internal' }, interval: '5m', exceptions_list: getListArrayMock(), related_integrations: [], diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts index 9546ab3a59b09..30fe84514e05a 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts @@ -266,13 +266,12 @@ describe('rule_source', () => { expect(result.data).toEqual(payload); }); - test('it should not validate a rule with "rule_source" set to undefined', () => { + test('it should validate a rule with "rule_source" set to undefined', () => { const payload = getRulesSchemaMock(); - // @ts-expect-error - delete payload.rule_source; + payload.rule_source = undefined; const result = RuleResponse.safeParse(payload); - expectParseError(result); - expect(stringifyZodError(result.error)).toMatchInlineSnapshot(`"rule_source: Required"`); + expectParseSuccess(result); + expect(result.data).toEqual(payload); }); }); From 6470dbda3f612d848c2fcad7c4664c8ab786e98f Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 17:19:37 -0500 Subject: [PATCH 067/120] Fix type error in converter Now that ruleSource is no longer a guaranteed field, we need to handle when it's not there. --- .../converters/convert_rule_response_to_alerting_rule.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts index ea8f49107f8b6..5d2b0889acf1f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_rule_response_to_alerting_rule.ts @@ -70,7 +70,7 @@ export const convertRuleResponseToAlertingRule = ( from: rule.from, investigationFields: rule.investigation_fields, immutable: rule.immutable, - ruleSource: convertObjectKeysToCamelCase(rule.rule_source), + ruleSource: rule.rule_source ? convertObjectKeysToCamelCase(rule.rule_source) : undefined, license: rule.license, outputIndex: rule.output_index ?? '', timelineId: rule.timeline_id, From eb7b6fd11fbfa23d251b80f09296e5b96231bebc Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 17:45:22 -0500 Subject: [PATCH 068/120] Fixing route unit tests * Ensure we're mocking the correct thing now that implementation's changed * Remove old/unnecessary arguments from test fn call --- .../api/rules/import_rules/route.test.ts | 50 ++++--------------- .../import/import_rules_with_source.test.ts | 3 +- .../logic/import/import_rules_with_source.ts | 2 - 3 files changed, 10 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts index e3588b5beae32..2ca46bc78b3e9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts @@ -56,7 +56,7 @@ describe('Import rules route', () => { clients.rulesClient.find.mockResolvedValue(getEmptyFindResult()); // no extant rules clients.rulesClient.update.mockResolvedValue(getRuleMock(getQueryRuleParams())); clients.detectionRulesClient.createCustomRule.mockResolvedValue(getRulesSchemaMock()); - clients.detectionRulesClient.importRule.mockResolvedValue(getRulesSchemaMock()); + clients.detectionRulesClient.legacyImportRule.mockResolvedValue(getRulesSchemaMock()); clients.actionsClient.getAll.mockResolvedValue([]); context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValue( elasticsearchClientMock.createSuccessTransportRequestPromise(getBasicEmptySearchResponse()) @@ -90,7 +90,7 @@ describe('Import rules route', () => { describe('unhappy paths', () => { test('returns a 403 error object if ML Authz fails', async () => { - clients.detectionRulesClient.importRule.mockImplementationOnce(async () => { + clients.detectionRulesClient.legacyImportRule.mockImplementationOnce(async () => { throw new HttpAuthzError('mocked validation message'); }); @@ -144,14 +144,17 @@ describe('Import rules route', () => { describe('with prebuilt rules customization enabled', () => { beforeEach(() => { + clients.detectionRulesClient.importRules.mockResolvedValueOnce([]); server = serverMock.create(); // old server already registered this route config = configMock.withExperimentalFeature(config, 'prebuiltRulesCustomizationEnabled'); importRulesRoute(server.router, config); }); - test('returns 500 if prebuilt rule installation fails', async () => { - mockPrebuiltRuleAssetsClient.fetchLatestAssets.mockRejectedValue(new Error('test error')); + test('returns 500 if importing fails', async () => { + clients.detectionRulesClient.importRules + .mockReset() + .mockRejectedValue(new Error('test error')); const response = await server.inject(request, requestContextMock.convertContext(context)); @@ -216,7 +219,7 @@ describe('Import rules route', () => { describe('rule with existing rule_id', () => { test('returns with reported conflict if `overwrite` is set to `false`', async () => { clients.rulesClient.find.mockResolvedValue(getFindResultWithSingleHit()); // extant rule - clients.detectionRulesClient.importRule.mockRejectedValue({ + clients.detectionRulesClient.legacyImportRule.mockRejectedValue({ message: 'rule_id: "rule-1" already exists', statusCode: 409, }); @@ -272,41 +275,6 @@ describe('Import rules route', () => { }); }); }); - - it('returns an error if rule is missing a version', async () => { - const ruleWithoutVersion = getImportRulesWithIdSchemaMock('rule-1'); - delete ruleWithoutVersion.version; - const payload = buildHapiStream(rulesToNdJsonString([ruleWithoutVersion])); - const versionlessRequest = getImportRulesRequest(payload); - - const response = await server.inject( - versionlessRequest, - requestContextMock.convertContext(context) - ); - - expect(response.status).toEqual(200); - expect(response.body).toEqual({ - errors: [ - { - error: { - message: `version: Required`, - status_code: 400, - }, - rule_id: '(unknown id)', - }, - ], - success: false, - success_count: 0, - rules_count: 1, - exceptions_errors: [], - exceptions_success: true, - exceptions_success_count: 0, - action_connectors_success: true, - action_connectors_success_count: 0, - action_connectors_warnings: [], - action_connectors_errors: [], - }); - }); }); describe('multi rule import', () => { @@ -469,7 +437,7 @@ describe('Import rules route', () => { }); test('returns with reported conflict if `overwrite` is set to `false`', async () => { - clients.detectionRulesClient.importRule.mockRejectedValueOnce({ + clients.detectionRulesClient.legacyImportRule.mockRejectedValueOnce({ message: 'rule_id: "rule-1" already exists', statusCode: 409, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts index 0c44d66d43c52..a9586939f3120 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts @@ -125,7 +125,7 @@ describe('importRules', () => { }; }); - it('imports a prebuilt rule when allowPrebuiltRules is true', async () => { + it('imports a prebuilt rule', async () => { clients.detectionRulesClient.importRule.mockResolvedValue({ ...getRulesSchemaMock(), rule_id: prebuiltRuleToImport.rule_id, @@ -137,7 +137,6 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - allowPrebuiltRules: true, prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, savedObjectsClient, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts index dce10b105be89..c7649e9ab91e4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts @@ -51,7 +51,6 @@ export const importRules = async ({ overwriteRules, detectionRulesClient, prebuiltRulesImportHelper, - allowPrebuiltRules, allowMissingConnectorSecrets, savedObjectsClient, }: { @@ -60,7 +59,6 @@ export const importRules = async ({ overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; prebuiltRulesImportHelper: PrebuiltRulesImportHelper; - allowPrebuiltRules?: boolean; allowMissingConnectorSecrets?: boolean; savedObjectsClient: SavedObjectsClientContract; }) => { From b3944dca227dcf0576a7f3a3844be36fa6f87bbd Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 21:19:20 -0500 Subject: [PATCH 069/120] Fix tests resulting from bugfix Some conversion code was changed in 17f68a39d28f217b59df5f9c867b94702af7dcb0 for the purposes of allowing rule_source and other optional fields. It also fixed a bug caused by commonParamsCamelToSnake where the `meta` object was left untouched, which is what these changed tests verify. --- .../lib/detection_engine/routes/__mocks__/utils.ts | 2 +- .../logic/export/get_export_all.test.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts index 687bf91655e2a..3d4a8105dc151 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts @@ -81,7 +81,7 @@ export const getOutputRuleAlertForRest = (): RuleResponse => ({ }, ], meta: { - someMeta: 'someField', + some_meta: 'someField', }, timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts index 382df4bfa5ffc..f60eedd0ee8b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts @@ -70,7 +70,7 @@ describe('getExportAll', () => { ...alert.params, filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], threat: getThreatMock(), - meta: { someMeta: 'someField' }, + meta: { some_meta: 'someField' }, timelineId: 'some-timeline-id', timelineTitle: 'some-timeline-title', }; @@ -121,7 +121,7 @@ describe('getExportAll', () => { setup: '', timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', - meta: { someMeta: 'someField' }, + meta: { some_meta: 'someField' }, severity: 'high', severity_mapping: [], updated_by: 'elastic', @@ -200,7 +200,7 @@ describe('getExportAll', () => { ...alert.params, filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], threat: getThreatMock(), - meta: { someMeta: 'someField' }, + meta: { some_meta: 'someField' }, timelineId: 'some-timeline-id', timelineTitle: 'some-timeline-title', }; @@ -303,7 +303,7 @@ describe('getExportAll', () => { setup: '', timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', - meta: { someMeta: 'someField' }, + meta: { some_meta: 'someField' }, severity: 'high', severity_mapping: [], updated_by: 'elastic', @@ -375,7 +375,7 @@ describe('getExportAll', () => { ...alert.params, filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], threat: getThreatMock(), - meta: { someMeta: 'someField' }, + meta: { some_meta: 'someField' }, timelineId: 'some-timeline-id', timelineTitle: 'some-timeline-title', }; From 187030e09ffdbbfe8ad8cbe8cdaebd3933a1f369 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 21:21:56 -0500 Subject: [PATCH 070/120] Remove immutable from rules stream tests Since RuleToImport no longer sets a default `immutable` value, the rules stream no longer has them. --- .../import/create_rules_stream_from_ndjson.test.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts index 5e37f161c3dde..2f5c384b97bd5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts @@ -66,7 +66,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }, { rule_id: 'rule-2', @@ -80,7 +79,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }, ]); }); @@ -135,7 +133,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }, { rule_id: 'rule-2', @@ -149,7 +146,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }, ]); }); @@ -184,7 +180,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }, { rule_id: 'rule-2', @@ -198,7 +193,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }, ]); }); @@ -232,7 +226,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }); expect(resultOrError[1].message).toEqual( `Expected property name or '}' in JSON at position 1` @@ -249,7 +242,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }); }); @@ -282,7 +274,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }); expect(resultOrError[1].message).toContain( `name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 1 more` @@ -299,7 +290,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }); }); @@ -358,7 +348,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, investigation_fields: { field_names: ['foo', 'bar'], }, @@ -375,7 +364,6 @@ describe('create_rules_stream_from_ndjson', () => { severity: 'low', interval: '5m', type: 'query', - immutable: false, }, ]); }); From 430495dd45fed44bd51854e382ca7f06d2af9875 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 21:29:17 -0500 Subject: [PATCH 071/120] Fix tests related to swapped implementations We were mocking the wrong methods and using the wrong inputs --- .../logic/import/import_rules_utils.test.ts | 4 ++-- .../logic/import/import_rules_with_source.test.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts index 5a582c28fda78..c240a4f82f26c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts @@ -59,7 +59,7 @@ describe('importRules', () => { }); it('returns 409 error if DetectionRulesClient throws with 409 - existing rule', async () => { - clients.detectionRulesClient.importRule.mockImplementationOnce(async () => { + clients.detectionRulesClient.legacyImportRule.mockImplementationOnce(async () => { throw createBulkErrorObject({ ruleId: ruleToImport.rule_id, statusCode: 409, @@ -88,7 +88,7 @@ describe('importRules', () => { }); it('creates rule if no matching existing rule found', async () => { - clients.detectionRulesClient.importRule.mockResolvedValue({ + clients.detectionRulesClient.legacyImportRule.mockResolvedValue({ ...getRulesSchemaMock(), rule_id: ruleToImport.rule_id, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts index a9586939f3120..9474eee0b189d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts @@ -7,7 +7,7 @@ import type { SavedObjectsClientContract } from '@kbn/core/server'; import { savedObjectsClientMock } from '@kbn/core/server/mocks'; -import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; +import { getValidatedRuleToImportMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; import { requestContextMock } from '../../../routes/__mocks__'; import { prebuiltRulesImportHelperMock } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper.mock'; @@ -17,7 +17,7 @@ import { createBulkErrorObject } from '../../../routes/utils'; describe('importRules', () => { const { clients, context } = requestContextMock.createTools(); - const ruleToImport = getImportRulesSchemaMock(); + const ruleToImport = getValidatedRuleToImportMock(); let savedObjectsClient: jest.Mocked; let mockPrebuiltRulesImportHelper: ReturnType; @@ -115,11 +115,11 @@ describe('importRules', () => { }); describe('compatibility with prebuilt rules', () => { - let prebuiltRuleToImport: ReturnType; + let prebuiltRuleToImport: ReturnType; beforeEach(() => { prebuiltRuleToImport = { - ...getImportRulesSchemaMock(), + ...getValidatedRuleToImportMock(), immutable: true, version: 1, }; From 18265b9f8577b1feb3b0550f0dde3afb02f9d6e4 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 13 Sep 2024 21:38:45 -0500 Subject: [PATCH 072/120] Remove outdated integration test This behavior was intended at the start of the feature, but we eventually decided against it in order to preserve backwards compatibility. --- .../import_rules.ts | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index b27187488cd73..1b9cc4af10020 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1683,24 +1683,6 @@ export default ({ getService }: FtrProviderContext): void => { }); }); - it('rejects rules without a version', async () => { - const rule = getCustomQueryRuleParams({}); - delete rule.version; - const ndjson = combineToNdJson(rule); - - const { body } = await supertest - .post(`${DETECTION_ENGINE_RULES_URL}/_import`) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '2023-10-31') - .attach('file', Buffer.from(ndjson), 'rules.ndjson') - .expect(200); - - expect(body.errors).toHaveLength(1); - expect(body.errors[0]).toMatchObject({ - error: { message: ': Required', status_code: 400 }, - }); - }); - describe('calculation of the rule_source fields', () => { it('calculates a version of 1 for custom rules'); From 52e879eac0ac6344b9362728644f5db34d01ac41 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 16 Sep 2024 20:40:59 -0500 Subject: [PATCH 073/120] Attempt to fix (or at least change) the cypress failure The output of this failed expectation is a giant red JSON and a giant green JSON. After manually inspecting them, they look identical except for key order. The hope is that this change will either fix the issue or provide a better failure, but I'm unable to even run cypress tests locally right now. --- .../rule_actions/import_export/export_rule.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts index cc270d41c10a6..6018d98ef8f57 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts @@ -71,7 +71,7 @@ describe('Export rules', { tags: ['@ess', '@serverless'] }, () => { it('exports a custom rule', function () { exportRule('Rule to export'); cy.wait('@bulk_action').then(({ response }) => { - cy.wrap(response?.body).should('eql', expectedExportedRule(this.ruleResponse)); + cy.wrap(response?.body).should('deep.equal', expectedExportedRule(this.ruleResponse)); cy.get(TOASTER_BODY).should('have.text', 'Successfully exported 1 of 1 rule.'); }); }); From 3ba18d4f55e9306628295629cefd238eeed934af Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 16 Sep 2024 21:05:58 -0500 Subject: [PATCH 074/120] Fix test failing due to copy change --- .../trial_license_complete_tier/import_rules.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 1b9cc4af10020..3d5327036ff4f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1596,7 +1596,7 @@ export default ({ getService }: FtrProviderContext): void => { error: { status_code: 400, message: - 'Importing prebuilt rules is not supported. To import this rule as a custom rule, remove its "immutable" property try again.', + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, first duplicate the rule and then export it. [rule_id: rule-immutable]', }, }, ], From d89759c17fc425e126faf9bf7cb4f28c1dbfd2dc Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 16 Sep 2024 21:44:52 -0500 Subject: [PATCH 075/120] Move our FF-dependent tests to the appropriate test suite I didn't realize that the Rules Management team had already created a test suite specifically for this epic's FF being enabled. Thanks, Dmitrii! --- .../import_rules.ts | 50 +++++++++++++++++++ .../trial_license_complete_tier/index.ts | 3 +- .../import_rules.ts | 23 --------- 3 files changed, 52 insertions(+), 24 deletions(-) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts new file mode 100644 index 0000000000000..16239f12bebc5 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; + +import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants'; +import { combineToNdJson, getCustomQueryRuleParams } from '../../../../utils'; +import { deleteAllRules } from '../../../../../../../common/utils/security_solution'; +import { FtrProviderContext } from '../../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + const log = getService('log'); + + describe('@ess @serverless @skipInServerlessMKI import_rules', () => { + beforeEach(async () => { + await deleteAllRules(supertest, log); + }); + + describe('calculation of the rule_source fields', () => { + // TODO if we do this, how do distinguish a custom rule from a prebuilt one? + it('calculates a version of 1 for custom rules'); + + it('rejects a prebuilt rule with an unspecified version', async () => { + const rule = getCustomQueryRuleParams(); + delete rule.version; + const ndjson = combineToNdJson(rule); + + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach('file', Buffer.from(ndjson), 'rules.ndjson') + .expect(200); + + expect(body.errors).toHaveLength(1); + expect(body.errors[0]).toMatchObject({ + error: { + message: 'Rules must specify a "version" to be imported. [rule_id: rule-1]', + status_code: 400, + }, + }); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts index 76a461d438463..8bf98956fd9cf 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/index.ts @@ -8,7 +8,8 @@ import { FtrProviderContext } from '../../../../../../ftr_provider_context'; export default ({ loadTestFile }: FtrProviderContext): void => { - describe('Rules Management - Prebuilt Rules - Update Prebuilt Rules Package', function () { + describe('Rules Management - Prebuilt Rules - Prebuilt Rule Customization', function () { loadTestFile(require.resolve('./is_customized_calculation')); + loadTestFile(require.resolve('./import_rules')); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 3d5327036ff4f..140c1d99ca3d8 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1682,29 +1682,6 @@ export default ({ getService }: FtrProviderContext): void => { }); }); }); - - describe('calculation of the rule_source fields', () => { - it('calculates a version of 1 for custom rules'); - - // TODO enable feature flag for this one - it.skip('rejects a prebuilt rule with an unspecified version', async () => { - const rule = getCustomQueryRuleParams(); - delete rule.version; - const ndjson = combineToNdJson(rule); - - const { body } = await supertest - .post(`${DETECTION_ENGINE_RULES_URL}/_import`) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '2023-10-31') - .attach('file', Buffer.from(ndjson), 'rules.ndjson') - .expect(200); - - expect(body.errors).toHaveLength(1); - expect(body.errors[0]).toMatchObject({ - error: { message: 'version: Required', status_code: 400 }, - }); - }); - }); }); }); }; From 8923f24b431f7d51e842ea9069e55ae5b9c1e127 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 16 Sep 2024 22:25:40 -0500 Subject: [PATCH 076/120] Remove unnecessary field overrides in import logic With the small modification to `applyRuleDefaults` here, now only the "update" path overwrites the customization fields that we need to preserve through import. I also added a comment indicating why we do this. --- .../detection_rules_client/mergers/apply_rule_defaults.ts | 6 ++++-- .../methods/import_rule_with_source.ts | 8 +++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts index 0263a60ab44ad..a1ed42c439295 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts @@ -44,7 +44,9 @@ export const RULE_DEFAULTS = { version: 1, }; -export function applyRuleDefaults(rule: RuleCreateProps & { immutable?: boolean }) { +export function applyRuleDefaults( + rule: RuleCreateProps & { immutable?: boolean; rule_source?: RuleSource } +) { const typeSpecificParams = setTypeSpecificDefaults(rule); const immutable = rule.immutable ?? false; @@ -54,7 +56,7 @@ export function applyRuleDefaults(rule: RuleCreateProps & { immutable?: boolean ...typeSpecificParams, rule_id: rule.rule_id ?? uuidv4(), immutable, - rule_source: convertImmutableToRuleSource(immutable), + rule_source: rule.rule_source ?? convertImmutableToRuleSource(immutable), required_fields: addEcsToRequiredFields(rule.required_fields), }; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts index 9cb1a115c3471..5611220f23e75 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts @@ -44,7 +44,6 @@ export const importRuleWithSource = async ({ mlAuthz, }: ImportRuleOptions): Promise => { const { ruleToImport, overwriteRules, allowMissingConnectorSecrets } = importRulePayload; - const { rule_source: ruleSource, immutable } = ruleToImport; await validateMlAuth(mlAuthz, ruleToImport.type); @@ -67,8 +66,9 @@ export const importRuleWithSource = async ({ existingRule, ruleUpdate: ruleToImport, }); - ruleWithUpdates.rule_source = ruleSource; - ruleWithUpdates.immutable = immutable; + // applyRuleUpdate prefers the existing rule's values for `rule_source` and `immutable`, but we want to use the importing rule's calculated values + ruleWithUpdates.immutable = ruleToImport.immutable; + ruleWithUpdates.rule_source = ruleToImport.rule_source; const updatedRule = await rulesClient.update({ id: existingRule.id, @@ -82,8 +82,6 @@ export const importRuleWithSource = async ({ } const ruleWithDefaults = applyRuleDefaults(ruleToImport); - ruleWithDefaults.rule_source = ruleSource; - ruleWithDefaults.immutable = immutable; const payload = { ...convertRuleResponseToAlertingRule(ruleWithDefaults, actionsClient), From 73e12686090509c0245d7141b31b7342956109e4 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 17 Sep 2024 21:13:14 -0500 Subject: [PATCH 077/120] Fix another test failing due to copy changes --- .../trial_license_complete_tier/import_rules.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts index 140c1d99ca3d8..038ed1787843a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/import_rules.ts @@ -1630,7 +1630,7 @@ export default ({ getService }: FtrProviderContext): void => { error: { status_code: 400, message: - 'Importing prebuilt rules is not supported. To import this rule as a custom rule, remove its "immutable" property try again.', + 'Importing prebuilt rules is not supported. To import this rule as a custom rule, first duplicate the rule and then export it. [rule_id: rule-immutable]', }, }, ], From 14cdb0e5222e719ede097479d8e0b34baefeb4ce Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 17 Sep 2024 21:37:39 -0500 Subject: [PATCH 078/120] Change order of export fields causing test failure I believe this field order changed as a consequence of 17f68a39d28f217b59df5f9c867b94702af7dcb0, particularly the ordering changes to `internalRuleToApiResponse`. Practically this has no effect on the system, but it did break this one brittle test. --- .../rule_actions/import_export/export_rule.cy.ts | 2 +- .../cypress/objects/rule.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts index 6018d98ef8f57..cc270d41c10a6 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts @@ -71,7 +71,7 @@ describe('Export rules', { tags: ['@ess', '@serverless'] }, () => { it('exports a custom rule', function () { exportRule('Rule to export'); cy.wait('@bulk_action').then(({ response }) => { - cy.wrap(response?.body).should('deep.equal', expectedExportedRule(this.ruleResponse)); + cy.wrap(response?.body).should('eql', expectedExportedRule(this.ruleResponse)); cy.get(TOASTER_BODY).should('have.text', 'Successfully exported 1 of 1 rule.'); }); }); diff --git a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts index 50e358515d922..86558c3892c98 100644 --- a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts @@ -576,26 +576,26 @@ export const expectedExportedRule = (ruleResponse: Cypress.Response Date: Wed, 18 Sep 2024 21:58:04 -0500 Subject: [PATCH 079/120] Define a new Error representing failures during rule import This will help us better delineate between errors from utility functions and their handling, by removing the status code from the error itself and instead placing that responsibility on the caller. In this instance, all these errors were being treated as 400s, but we also represent a "conflict" error, so I've added a `type` field to allow that plus some extensibility. --- .../check_rule_exception_references.test.ts | 12 ++--- .../import/check_rule_exception_references.ts | 10 ++-- .../rule_management/logic/import/errors.ts | 49 +++++++++++++++++++ .../logic/import/import_rules_utils.ts | 10 +++- 4 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.test.ts index 2a249e7d9383a..b6f9c8959fb77 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.test.ts @@ -63,11 +63,11 @@ describe('checkRuleExceptionReferences', () => { [ { error: { + ruleId: 'rule-1', message: 'Rule with rule_id: "rule-1" references a non existent exception list of list_id: "my-list". Reference has been removed.', - status_code: 400, + type: 'unknown', }, - rule_id: 'rule-1', }, ], [], @@ -94,11 +94,11 @@ describe('checkRuleExceptionReferences', () => { [ { error: { + ruleId: 'rule-1', message: 'Rule with rule_id: "rule-1" references a non existent exception list of list_id: "my-list". Reference has been removed.', - status_code: 400, + type: 'unknown', }, - rule_id: 'rule-1', }, ], [], @@ -127,9 +127,9 @@ describe('checkRuleExceptionReferences', () => { error: { message: 'Rule with rule_id: "rule-1" references a non existent exception list of list_id: "my-list". Reference has been removed.', - status_code: 400, + ruleId: 'rule-1', + type: 'unknown', }, - rule_id: 'rule-1', }, ], [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts index efa6026d875bf..9e54cc1ee816a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts @@ -7,8 +7,7 @@ import type { ListArray, ExceptionListSchema } from '@kbn/securitysolution-io-ts-list-types'; import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; -import type { BulkError } from '../../../routes/utils'; -import { createBulkErrorObject } from '../../../routes/utils'; +import { type RuleImportError, createRuleImportError } from './errors'; /** * Helper to check if all the exception lists referenced on a @@ -27,9 +26,9 @@ export const checkRuleExceptionReferences = ({ }: { rule: RuleToImport; existingLists: Record; -}): [BulkError[], ListArray] => { +}): [RuleImportError[], ListArray] => { let ruleExceptions: ListArray = []; - let errors: BulkError[] = []; + let errors: RuleImportError[] = []; const { rule_id: ruleId } = rule; const exceptionLists = rule.exceptions_list ?? []; @@ -54,9 +53,8 @@ export const checkRuleExceptionReferences = ({ // this error to notify a user of the action taken. errors = [ ...errors, - createBulkErrorObject({ + createRuleImportError({ ruleId, - statusCode: 400, message: `Rule with rule_id: "${ruleId}" references a non existent exception list of list_id: "${exceptionList.list_id}". Reference has been removed.`, }), ]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts new file mode 100644 index 0000000000000..2d81aa56a138a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { has } from 'lodash'; + +export type RuleImportErrorType = 'conflict' | 'unknown'; + +/** + * Generic interface representing a server-side failure during rule import. + * Used by utilities that import rules or related entities. + * + * NOTE that this does not inherit from Error + */ +export interface RuleImportError { + error: { + ruleId: string; + message: string; + type: RuleImportErrorType; + }; +} + +export const createRuleImportError = ({ + ruleId, + message, + type, +}: { + ruleId: string; + message: string; + type?: RuleImportErrorType; +}): RuleImportError => ({ + error: { + ruleId, + message, + type: type ?? 'unknown', + }, +}); + +export const isRuleImportError = (obj: unknown): obj is RuleImportError => + has(obj, 'error') && + has(obj, 'error.ruleId') && + has(obj, 'error.type') && + has(obj, 'error.message'); + +export const isRuleConflictError = (error: RuleImportError): boolean => + error.error.type === 'conflict'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 9ce96239f753d..dd8ce5a20a811 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -110,7 +110,15 @@ export const importRules = async ({ existingLists, }); - importRuleResponse = [...importRuleResponse, ...exceptionErrors]; + const exceptionBulkErrors = exceptionErrors.map((error) => + createBulkErrorObject({ + ruleId: error.ruleId, + statusCode: 400, + message: error.error.message, + }) + ); + + importRuleResponse = [...importRuleResponse, ...exceptionBulkErrors]; const importedRule = await detectionRulesClient.legacyImportRule({ ruleToImport: { From 5c5e6371ef4fc1737068c0af131a1f212dbc60ca Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 18 Sep 2024 23:35:29 -0500 Subject: [PATCH 080/120] WIP: Separating route-specific logic from general importing This is the first pass of keeping any route-specific logic out of the DetectionRulesClient, and particularly the new `importRules` method that's added in this effort. It removes route concepts like streams and status codes and responses (except RuleResponse) from the client method, and moves them into an intermediate function that then calls the client. I'm also leveraging the new `RuleImportError` here, and `importRules` now returns `Array`, which feels a lot better. At this point, tests are very much broken, so once this idea is vetted I'll be fixing those. --- .../api/rules/import_rules/route.ts | 8 +- .../detection_rules_client.ts | 6 +- .../detection_rules_client_interface.ts | 9 +- .../methods/import_rule_with_source.ts | 6 +- .../methods/import_rules.ts | 108 ++++++++++++++ .../logic/import/import_rules_utils.ts | 2 +- .../logic/import/import_rules_with_source.ts | 141 +++++------------- 7 files changed, 163 insertions(+), 117 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index c5b996fe4a97b..57c82e27d36f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -23,6 +23,7 @@ import type { BulkError, ImportRuleResponse } from '../../../../routes/utils'; import { buildSiemResponse, isBulkError, isImportRegular } from '../../../../routes/utils'; import { PrebuiltRulesImportHelper } from '../../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; +import { importRules } from '../../../logic/import/import_rules_with_source'; import { importRules as legacyImportRules } from '../../../logic/import/import_rules_utils'; import { createRulesAndExceptionsStreamFromNdJson } from '../../../logic/import/create_rules_stream_from_ndjson'; import type { RuleExceptionsPromiseFromStreams } from '../../../logic/import/import_rules_utils'; @@ -155,12 +156,13 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C let importRuleResponse: ImportRuleResponse[] = []; if (prebuiltRulesCustomizationEnabled) { - importRuleResponse = await detectionRulesClient.importRules({ + importRuleResponse = await importRules({ ruleChunks: chunkParseObjects, rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], overwriteRules: request.query.overwrite, allowMissingConnectorSecrets: !!actionConnectors.length, prebuiltRulesImportHelper, + detectionRulesClient, savedObjectsClient, }); } else { @@ -182,7 +184,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C return false; } }); - const importRules: ImportRulesResponse = { + const importedRules: ImportRulesResponse = { success: errorsResp.length === 0, success_count: successes.length, rules_count: rules.length, @@ -196,7 +198,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C action_connectors_warnings: actionConnectorWarnings, }; - return response.ok({ body: ImportRulesResponse.parse(importRules) }); + return response.ok({ body: ImportRulesResponse.parse(importedRules) }); } catch (err) { const error = transformError(err); return siemResponse.error({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts index d7dc069d2b6cd..7fa2272e40fc9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts @@ -13,8 +13,7 @@ import type { RuleResponse } from '../../../../../../common/api/detection_engine import { withSecuritySpan } from '../../../../../utils/with_security_span'; import type { MlAuthz } from '../../../../machine_learning/authz'; import { createPrebuiltRuleAssetsClient } from '../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; -import type { ImportRuleResponse } from '../../../routes/utils'; -import { importRules } from '../import/import_rules_with_source'; +import type { RuleImportError } from '../import/errors'; import type { CreateCustomRuleArgs, CreatePrebuiltRuleArgs, @@ -34,6 +33,7 @@ import { patchRule } from './methods/patch_rule'; import { updateRule } from './methods/update_rule'; import { upgradePrebuiltRule } from './methods/upgrade_prebuilt_rule'; import { importRuleWithSource } from './methods/import_rule_with_source'; +import { importRules } from './methods/import_rules'; interface DetectionRulesClientParams { actionsClient: ActionsClient; @@ -149,7 +149,7 @@ export const createDetectionRulesClient = ({ }); }, - async importRules(args: ImportRulesArgs): Promise { + async importRules(args: ImportRulesArgs): Promise> { return withSecuritySpan('DetectionRulesClient.importRules', async () => { return importRules({ ...args, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts index 16dbbd98d9b42..de78d7fdf2665 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts @@ -16,10 +16,9 @@ import type { RuleToImport, RuleSource, } from '../../../../../../common/api/detection_engine'; +import type { RuleImportError } from '../import/errors'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; -import type { ImportRuleResponse } from '../../../routes/utils'; -import type { PromiseFromStreams } from '../import/import_rules_utils'; export interface IDetectionRulesClient { createCustomRule: (args: CreateCustomRuleArgs) => Promise; @@ -30,7 +29,7 @@ export interface IDetectionRulesClient { upgradePrebuiltRule: (args: UpgradePrebuiltRuleArgs) => Promise; legacyImportRule: (args: LegacyImportRuleArgs) => Promise; importRule: (args: ImportRuleArgs) => Promise; - importRules: (args: ImportRulesArgs) => Promise; + importRules: (args: ImportRulesArgs) => Promise>; } export interface CreateCustomRuleArgs { @@ -70,8 +69,8 @@ export interface ImportRuleArgs { } export interface ImportRulesArgs { - ruleChunks: PromiseFromStreams[][]; - rulesResponseAcc: ImportRuleResponse[]; + rules: RuleToImport[]; + detectionRulesClient: IDetectionRulesClient; overwriteRules: boolean; prebuiltRulesImportHelper: PrebuiltRulesImportHelper; allowMissingConnectorSecrets?: boolean; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts index 5611220f23e75..1ac02ee63c0b2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts @@ -12,7 +12,6 @@ import { ruleTypeMappings } from '@kbn/securitysolution-rules'; import type { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; import type { MlAuthz } from '../../../../../machine_learning/authz'; import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; -import { createBulkErrorObject } from '../../../../routes/utils'; import { convertAlertingRuleToRuleResponse } from '../converters/convert_alerting_rule_to_rule_response'; import { convertRuleResponseToAlertingRule } from '../converters/convert_rule_response_to_alerting_rule'; import type { ImportRuleArgs } from '../detection_rules_client_interface'; @@ -21,6 +20,7 @@ import { applyRuleDefaults } from '../mergers/apply_rule_defaults'; import { validateMlAuth, toggleRuleEnabledOnUpdate } from '../utils'; import { getRuleByRuleId } from './get_rule_by_rule_id'; import { SERVER_APP_ID } from '../../../../../../../common'; +import { createRuleImportError } from '../../import/errors'; interface ImportRuleOptions { actionsClient: ActionsClient; @@ -53,9 +53,9 @@ export const importRuleWithSource = async ({ }); if (existingRule && !overwriteRules) { - throw createBulkErrorObject({ + throw createRuleImportError({ ruleId: existingRule.rule_id, - statusCode: 409, + type: 'conflict', message: `rule_id: "${existingRule.rule_id}" already exists`, }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts new file mode 100644 index 0000000000000..0dcc8912b996f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; + +import type { RuleResponse, RuleToImport } from '../../../../../../../common/api/detection_engine'; +import { ruleToImportHasVersion } from '../../../../../../../common/api/detection_engine/rule_management'; +import type { PrebuiltRulesImportHelper } from '../../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; +import { + type RuleImportError, + createRuleImportError, + isRuleImportError, +} from '../../import/errors'; +import { checkRuleExceptionReferences } from '../../import/check_rule_exception_references'; +import { calculateRuleSourceForImport } from '../../import/calculate_rule_source_for_import'; +import { getReferencedExceptionLists } from '../../import/gather_referenced_exceptions'; +import type { IDetectionRulesClient } from '../detection_rules_client_interface'; + +/** + * Imports rules + */ + +export const importRules = async ({ + allowMissingConnectorSecrets, + detectionRulesClient, + overwriteRules, + prebuiltRulesImportHelper, + rules, + savedObjectsClient, +}: { + allowMissingConnectorSecrets?: boolean; + detectionRulesClient: IDetectionRulesClient; + overwriteRules: boolean; + prebuiltRulesImportHelper: PrebuiltRulesImportHelper; + rules: RuleToImport[]; + savedObjectsClient: SavedObjectsClientContract; +}): Promise> => { + const existingLists = await getReferencedExceptionLists({ + rules, + savedObjectsClient, + }); + const prebuiltRuleAssets = await prebuiltRulesImportHelper.fetchMatchingAssets({ + rules, + }); + + const installedRuleIds = await prebuiltRulesImportHelper.fetchAssetRuleIds({ + rules, + }); + + return Promise.all( + rules.map(async (rule) => { + if (!ruleToImportHasVersion(rule)) { + return createRuleImportError({ + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.cannotImportRuleWithoutVersion', + { + defaultMessage: 'Rules must specify a "version" to be imported. [rule_id: {ruleId}]', + values: { ruleId: rule.rule_id }, + } + ), + ruleId: rule.rule_id, + }); + } + + const { immutable, ruleSource } = calculateRuleSourceForImport({ + rule, + prebuiltRuleAssets, + installedRuleIds, + }); + + try { + const [exceptionErrors, exceptions] = checkRuleExceptionReferences({ + rule, + existingLists, + }); + + const importedRule = await detectionRulesClient.importRule({ + ruleToImport: { + ...rule, + rule_source: ruleSource, + immutable, + exceptions_list: [...exceptions], + }, + overwriteRules, + allowMissingConnectorSecrets, + }); + + return [...exceptionErrors, importedRule]; + } catch (err) { + const { error, message } = err; + + if (isRuleImportError(err)) { + return err; + } + + return createRuleImportError({ + ruleId: rule.rule_id, + message: message ?? error?.message ?? 'unknown error', + }); + } + }) + ).then((results) => results.flat()); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index dd8ce5a20a811..a3f69882c9e20 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -112,7 +112,7 @@ export const importRules = async ({ const exceptionBulkErrors = exceptionErrors.map((error) => createBulkErrorObject({ - ruleId: error.ruleId, + ruleId: error.error.ruleId, statusCode: 400, message: error.error.message, }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts index c7649e9ab91e4..eabd903f97d16 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts @@ -5,24 +5,18 @@ * 2.0. */ -import { i18n } from '@kbn/i18n'; +import { partition } from 'lodash'; import type { SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; import type { ImportExceptionsListSchema, ImportExceptionListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import { - ruleToImportHasVersion, - type RuleToImport, -} from '../../../../../../common/api/detection_engine/rule_management'; -import type { ImportRuleResponse } from '../../../routes/utils'; -import { createBulkErrorObject } from '../../../routes/utils'; +import { type RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; +import { type ImportRuleResponse, createBulkErrorObject } from '../../../routes/utils'; import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; -import { checkRuleExceptionReferences } from './check_rule_exception_references'; -import { calculateRuleSourceForImport } from './calculate_rule_source_for_import'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; -import { getReferencedExceptionLists } from './gather_referenced_exceptions'; +import { isRuleConflictError, isRuleImportError } from './errors'; export type PromiseFromStreams = RuleToImport | Error; export interface RuleExceptionsPromiseFromStreams { @@ -31,8 +25,10 @@ export interface RuleExceptionsPromiseFromStreams { actionConnectors: SavedObject[]; } +const ruleIsError = (rule: RuleToImport | Error): rule is Error => rule instanceof Error; + /** - * Takes rules to be imported and either creates or updates rules + * Takes a stream of rules to be imported and either creates or updates rules * based on user overwrite preferences * @param ruleChunks {array} - rules being imported * @param rulesResponseAcc {array} - the accumulation of success and @@ -62,112 +58,53 @@ export const importRules = async ({ allowMissingConnectorSecrets?: boolean; savedObjectsClient: SavedObjectsClientContract; }) => { - let importRuleResponse: ImportRuleResponse[] = [...rulesResponseAcc]; + let response: ImportRuleResponse[] = [...rulesResponseAcc]; // If we had 100% errors and no successful rule could be imported we still have to output an error. // otherwise we would output we are success importing 0 rules. if (ruleChunks.length === 0) { - return importRuleResponse; + return response; } await prebuiltRulesImportHelper.setup(); while (ruleChunks.length) { - const batchParseObjects = ruleChunks.shift() ?? []; - const existingLists = await getReferencedExceptionLists({ - rules: batchParseObjects, + const ruleChunk = ruleChunks.shift() ?? []; + const [errors, rules] = partition(ruleChunk, ruleIsError); + + const importedRulesResponse = await detectionRulesClient.importRules({ + allowMissingConnectorSecrets, + detectionRulesClient, + overwriteRules, + prebuiltRulesImportHelper, + rules, savedObjectsClient, }); - const prebuiltRuleAssets = await prebuiltRulesImportHelper.fetchMatchingAssets({ - rules: batchParseObjects, - }); - const installedRuleIds = await prebuiltRulesImportHelper.fetchAssetRuleIds({ - rules: batchParseObjects, - }); - - const newImportRuleResponse = await Promise.all( - batchParseObjects.reduce>>((accum, parsedRule) => { - const importsWorkerPromise = new Promise(async (resolve, reject) => { - try { - if (parsedRule instanceof Error) { - // If the JSON object had a validation or parse error then we return - // early with the error and an (unknown) for the ruleId - resolve( - createBulkErrorObject({ - statusCode: 400, - message: parsedRule.message, - }) - ); - return null; - } - - if (!ruleToImportHasVersion(parsedRule)) { - resolve( - createBulkErrorObject({ - statusCode: 400, - message: i18n.translate( - 'xpack.securitySolution.detectionEngine.rules.cannotImportRuleWithoutVersion', - { - defaultMessage: - 'Rules must specify a "version" to be imported. [rule_id: {ruleId}]', - values: { ruleId: parsedRule.rule_id }, - } - ), - ruleId: parsedRule.rule_id, - }) - ); - - return null; - } - const { immutable, ruleSource } = calculateRuleSourceForImport({ - rule: parsedRule, - prebuiltRuleAssets, - installedRuleIds, - }); - - try { - const [exceptionErrors, exceptions] = checkRuleExceptionReferences({ - rule: parsedRule, - existingLists, - }); + const genericErrors = errors.map((error) => + createBulkErrorObject({ + statusCode: 400, + message: error.message, + }) + ); - importRuleResponse = [...importRuleResponse, ...exceptionErrors]; + const importResponses = importedRulesResponse.map((rule) => { + if (isRuleImportError(rule)) { + return createBulkErrorObject({ + message: rule.error.message, + statusCode: isRuleConflictError(rule) ? 409 : 400, + ruleId: rule.error.ruleId, + }); + } - const importedRule = await detectionRulesClient.importRule({ - ruleToImport: { - ...parsedRule, - rule_source: ruleSource, - immutable, - exceptions_list: [...exceptions], - }, - overwriteRules, - allowMissingConnectorSecrets, - }); + return { + rule_id: rule.rule_id, + status_code: 200, + }; + }); - resolve({ - rule_id: importedRule.rule_id, - status_code: 200, - }); - } catch (err) { - const { error, statusCode, message } = err; - resolve( - createBulkErrorObject({ - ruleId: parsedRule.rule_id, - statusCode: statusCode ?? error?.status_code ?? 400, - message: message ?? error?.message ?? 'unknown error', - }) - ); - } - } catch (error) { - reject(error); - } - }); - return [...accum, importsWorkerPromise]; - }, []) - ); - importRuleResponse = [...importRuleResponse, ...newImportRuleResponse]; + response = [...response, ...genericErrors, ...importResponses]; } - return importRuleResponse; + return response; }; From df456f24ae555c0e4ae6abeaff6b6745541e77a1 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 19 Sep 2024 13:37:39 -0500 Subject: [PATCH 081/120] Clean up some interfaces by combining two helper utilities As evidenced by the unit tests, we were never calling `createRulesStreamFromNdJson` without also calling `createPromiseFromStreams` to get a workable promise. As a consequence of calling these separately, callers had to manually cast the result of `createPromiseFromStreams`, which made for a lot of unnecessary verbosity. We now instead simply call `createPromiseFromRuleImportStream`, and downstream functions can now deal with the simpler shared type `RuleFromImportStream` directly. --- .../api/rules/import_rules/route.test.ts | 6 +- .../api/rules/import_rules/route.ts | 11 +- ...te_promise_from_rule_import_stream.test.ts | 366 +++++++++++++++++ .../create_promise_from_rule_import_stream.ts | 39 ++ .../create_rules_stream_from_ndjson.test.ts | 371 ------------------ .../logic/import/import_rules_utils.ts | 17 +- .../logic/import/import_rules_with_source.ts | 18 +- .../rule_management/logic/import/types.ts | 10 + .../rule_management/utils/utils.test.ts | 116 +++--- 9 files changed, 485 insertions(+), 469 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/types.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts index 2ca46bc78b3e9..5d4dbec256172 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts @@ -26,7 +26,7 @@ import { getBasicEmptySearchResponse, } from '../../../../routes/__mocks__/request_responses'; -import * as createRulesAndExceptionsStreamFromNdJson from '../../../logic/import/create_rules_stream_from_ndjson'; +import * as createPromiseFromRuleImportStream from '../../../logic/import/create_promise_from_rule_import_stream'; import { getQueryRuleParams } from '../../../../rule_schema/mocks'; import { importRulesRoute } from './route'; import { HttpAuthzError } from '../../../../../machine_learning/validation'; @@ -120,9 +120,9 @@ describe('Import rules route', () => { }); }); - test('returns error if createRulesAndExceptionsStreamFromNdJson throws error', async () => { + test('returns error if createPromiseFromRuleImportStream throws error', async () => { const transformMock = jest - .spyOn(createRulesAndExceptionsStreamFromNdJson, 'createRulesAndExceptionsStreamFromNdJson') + .spyOn(createPromiseFromRuleImportStream, 'createPromiseFromRuleImportStream') .mockImplementation(() => { throw new Error('Test error'); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 57c82e27d36f2..e46a1436940d6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -8,7 +8,6 @@ import { schema } from '@kbn/config-schema'; import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import { createPromiseFromStreams } from '@kbn/utils'; import { chunk } from 'lodash/fp'; import { extname } from 'path'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; @@ -25,8 +24,7 @@ import { PrebuiltRulesImportHelper } from '../../../../prebuilt_rules/logic/preb import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; import { importRules } from '../../../logic/import/import_rules_with_source'; import { importRules as legacyImportRules } from '../../../logic/import/import_rules_utils'; -import { createRulesAndExceptionsStreamFromNdJson } from '../../../logic/import/create_rules_stream_from_ndjson'; -import type { RuleExceptionsPromiseFromStreams } from '../../../logic/import/import_rules_utils'; +import { createPromiseFromRuleImportStream } from '../../../logic/import/create_promise_from_rule_import_stream'; import { importRuleExceptions } from '../../../logic/import/import_rule_exceptions'; import { getTupleDuplicateErrorsAndUniqueRules, @@ -98,10 +96,9 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C const objectLimit = config.maxRuleImportExportSize; // parse file to separate out exceptions from rules - const readAllStream = createRulesAndExceptionsStreamFromNdJson(objectLimit); - const [{ exceptions, rules, actionConnectors }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([request.body.file as HapiReadableStream, ...readAllStream]); + const [{ exceptions, rules, actionConnectors }] = await createPromiseFromRuleImportStream( + { stream: request.body.file as HapiReadableStream, objectLimit } + ); // import exceptions, includes validation const { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts new file mode 100644 index 0000000000000..8e0bacbd5206a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts @@ -0,0 +1,366 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Readable } from 'stream'; +import { BadRequestError } from '@kbn/securitysolution-es-utils'; + +import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; +import { + getOutputDetailsSample, + getSampleDetailsAsNdjson, +} from '../../../../../../common/api/detection_engine/rule_management/mocks'; +import type { InvestigationFields } from '../../../../../../common/api/detection_engine'; +import { createPromiseFromRuleImportStream } from './create_promise_from_rule_import_stream'; + +export const getOutputSample = (): Partial => ({ + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', +}); + +export const getSampleAsNdjson = (sample: Partial): string => { + return `${JSON.stringify(sample)}\n`; +}; + +describe('createPromiseFromRuleImportStream', () => { + test('transforms an ndjson stream into a stream of rule objects', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + + expect(result).toEqual([ + { + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + { + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + ]); + }); + + // TODO - Yara - there's a integration test testing this, but causing timeouts here + test.skip('returns error when ndjson stream is larger than limit', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + }, + }); + + await expect( + createPromiseFromRuleImportStream({ stream: ndJsonStream, objectLimit: 2 }) + ).rejects.toThrowError("Can't import more than 1 rules"); + }); + + test('skips empty lines', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push('\n'); + this.push(getSampleAsNdjson(sample2)); + this.push(''); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + + expect(result).toEqual([ + { + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + { + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + ]); + }); + + test('filters the export details entry from the stream', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + const details = getOutputDetailsSample({ totalCount: 1, rulesCount: 1 }); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + this.push(getSampleDetailsAsNdjson(details)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + + expect(result).toEqual([ + { + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + { + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + ]); + }); + + test('handles non parsable JSON strings and inserts the error as part of the return array', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push('{,,,,\n'); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + + const resultOrError = result as Error[]; + expect(resultOrError[0]).toEqual({ + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }); + expect(resultOrError[1].message).toEqual(`Expected property name or '}' in JSON at position 1`); + expect(resultOrError[2]).toEqual({ + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }); + }); + + test('handles non-validated data', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(`{}\n`); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + const resultOrError = result as BadRequestError[]; + expect(resultOrError[0]).toEqual({ + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }); + expect(resultOrError[1].message).toContain( + `name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 1 more` + ); + expect(resultOrError[2]).toEqual({ + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }); + }); + + test('non validated data is an instanceof BadRequestError', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(`{}\n`); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + const resultOrError = result as BadRequestError[]; + expect(resultOrError[1] instanceof BadRequestError).toEqual(true); + }); + + test('migrates investigation_fields', async () => { + const sample1 = { + ...getOutputSample(), + investigation_fields: ['foo', 'bar'] as unknown as InvestigationFields, + }; + const sample2 = { + ...getOutputSample(), + rule_id: 'rule-2', + investigation_fields: [] as unknown as InvestigationFields, + }; + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const [{ rules: result }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); + expect(result).toEqual([ + { + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + investigation_fields: { + field_names: ['foo', 'bar'], + }, + }, + { + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + }, + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts new file mode 100644 index 0000000000000..19a77db0e0c86 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Readable } from 'stream'; +import { createPromiseFromStreams } from '@kbn/utils'; +import type { SavedObject } from '@kbn/core/server'; +import type { + ImportExceptionsListSchema, + ImportExceptionListItemSchema, +} from '@kbn/securitysolution-io-ts-list-types'; + +import { createRulesAndExceptionsStreamFromNdJson } from './create_rules_stream_from_ndjson'; +import type { RuleFromImportStream } from './types'; + +export interface RuleImportStreamResult { + rules: RuleFromImportStream[]; + exceptions: Array; + actionConnectors: SavedObject[]; +} + +/** + * Utility for generating a promise from a Readable stream corresponding to an + * NDJSON file. Used during rule import. + */ +export const createPromiseFromRuleImportStream = ({ + objectLimit, + stream, +}: { + objectLimit: number; + stream: Readable; +}): Promise => { + const readAllStream = createRulesAndExceptionsStreamFromNdJson(objectLimit); + + return createPromiseFromStreams([stream, ...readAllStream]); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts deleted file mode 100644 index 2f5c384b97bd5..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts +++ /dev/null @@ -1,371 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Readable } from 'stream'; -import { createPromiseFromStreams } from '@kbn/utils'; -import { createRulesAndExceptionsStreamFromNdJson } from './create_rules_stream_from_ndjson'; -import { BadRequestError } from '@kbn/securitysolution-es-utils'; - -import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; -import { - getOutputDetailsSample, - getSampleDetailsAsNdjson, -} from '../../../../../../common/api/detection_engine/rule_management/mocks'; -import type { RuleExceptionsPromiseFromStreams } from './import_rules_utils'; -import type { InvestigationFields } from '../../../../../../common/api/detection_engine'; - -export const getOutputSample = (): Partial => ({ - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', -}); - -export const getSampleAsNdjson = (sample: Partial): string => { - return `${JSON.stringify(sample)}\n`; -}; - -describe('create_rules_stream_from_ndjson', () => { - describe('createRulesAndExceptionsStreamFromNdJson', () => { - test('transforms an ndjson stream into a stream of rule objects', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - expect(result).toEqual([ - { - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }, - { - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }, - ]); - }); - - // TODO - Yara - there's a integration test testing this, but causing timeoutes here - test.skip('returns error when ndjson stream is larger than limit', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(2); - await expect( - createPromiseFromStreams([ - ndJsonStream, - ...rulesObjectsStream, - ]) - ).rejects.toThrowError("Can't import more than 1 rules"); - }); - - test('skips empty lines', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push('\n'); - this.push(getSampleAsNdjson(sample2)); - this.push(''); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - expect(result).toEqual([ - { - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }, - { - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }, - ]); - }); - - test('filters the export details entry from the stream', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - const details = getOutputDetailsSample({ totalCount: 1, rulesCount: 1 }); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - this.push(getSampleDetailsAsNdjson(details)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - expect(result).toEqual([ - { - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }, - { - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }, - ]); - }); - - test('handles non parsable JSON strings and inserts the error as part of the return array', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push('{,,,,\n'); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - const resultOrError = result as Error[]; - expect(resultOrError[0]).toEqual({ - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }); - expect(resultOrError[1].message).toEqual( - `Expected property name or '}' in JSON at position 1` - ); - expect(resultOrError[2]).toEqual({ - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }); - }); - - test('handles non-validated data', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(`{}\n`); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - const resultOrError = result as BadRequestError[]; - expect(resultOrError[0]).toEqual({ - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }); - expect(resultOrError[1].message).toContain( - `name: Required, description: Required, risk_score: Required, severity: Required, type: Invalid discriminator value. Expected 'eql' | 'query' | 'saved_query' | 'threshold' | 'threat_match' | 'machine_learning' | 'new_terms' | 'esql', and 1 more` - ); - expect(resultOrError[2]).toEqual({ - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }); - }); - - test('non validated data is an instanceof BadRequestError', async () => { - const sample1 = getOutputSample(); - const sample2 = getOutputSample(); - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(`{}\n`); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - const resultOrError = result as BadRequestError[]; - expect(resultOrError[1] instanceof BadRequestError).toEqual(true); - }); - - test('migrates investigation_fields', async () => { - const sample1 = { - ...getOutputSample(), - investigation_fields: ['foo', 'bar'] as unknown as InvestigationFields, - }; - const sample2 = { - ...getOutputSample(), - rule_id: 'rule-2', - investigation_fields: [] as unknown as InvestigationFields, - }; - sample2.rule_id = 'rule-2'; - const ndJsonStream = new Readable({ - read() { - this.push(getSampleAsNdjson(sample1)); - this.push(getSampleAsNdjson(sample2)); - this.push(null); - }, - }); - const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); - const [{ rules: result }] = await createPromiseFromStreams< - RuleExceptionsPromiseFromStreams[] - >([ndJsonStream, ...rulesObjectsStream]); - expect(result).toEqual([ - { - rule_id: 'rule-1', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - investigation_fields: { - field_names: ['foo', 'bar'], - }, - }, - { - rule_id: 'rule-2', - output_index: '.siem-signals', - risk_score: 50, - description: 'some description', - from: 'now-5m', - to: 'now', - index: ['index-1'], - name: 'some-name', - severity: 'low', - interval: '5m', - type: 'query', - }, - ]); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index a3f69882c9e20..435ae5e722f79 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -6,25 +6,14 @@ */ import { i18n } from '@kbn/i18n'; -import type { SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; -import type { - ImportExceptionsListSchema, - ImportExceptionListItemSchema, -} from '@kbn/securitysolution-io-ts-list-types'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; -import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; import { checkRuleExceptionReferences } from './check_rule_exception_references'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { getReferencedExceptionLists } from './gather_referenced_exceptions'; - -export type PromiseFromStreams = RuleToImport | Error; -export interface RuleExceptionsPromiseFromStreams { - rules: PromiseFromStreams[]; - exceptions: Array; - actionConnectors: SavedObject[]; -} +import type { RuleFromImportStream } from './types'; /** * Takes rules to be imported and either creates or updates rules @@ -48,7 +37,7 @@ export const importRules = async ({ allowMissingConnectorSecrets, savedObjectsClient, }: { - ruleChunks: PromiseFromStreams[][]; + ruleChunks: RuleFromImportStream[][]; rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts index eabd903f97d16..f8cc2431466e0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts @@ -6,26 +6,14 @@ */ import { partition } from 'lodash'; -import type { SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; -import type { - ImportExceptionsListSchema, - ImportExceptionListItemSchema, -} from '@kbn/securitysolution-io-ts-list-types'; -import { type RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; import { type ImportRuleResponse, createBulkErrorObject } from '../../../routes/utils'; import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { isRuleConflictError, isRuleImportError } from './errors'; +import type { RuleFromImportStream } from './types'; -export type PromiseFromStreams = RuleToImport | Error; -export interface RuleExceptionsPromiseFromStreams { - rules: PromiseFromStreams[]; - exceptions: Array; - actionConnectors: SavedObject[]; -} - -const ruleIsError = (rule: RuleToImport | Error): rule is Error => rule instanceof Error; +const ruleIsError = (rule: RuleFromImportStream): rule is Error => rule instanceof Error; /** * Takes a stream of rules to be imported and either creates or updates rules @@ -50,7 +38,7 @@ export const importRules = async ({ allowMissingConnectorSecrets, savedObjectsClient, }: { - ruleChunks: PromiseFromStreams[][]; + ruleChunks: RuleFromImportStream[][]; rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/types.ts new file mode 100644 index 0000000000000..b9bae0b23223b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/types.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; + +export type RuleFromImportStream = RuleToImport | Error; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts index 53e96dc627080..01cd024b86b0c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts @@ -7,7 +7,6 @@ import { partition } from 'lodash/fp'; import { Readable } from 'stream'; -import { createPromiseFromStreams } from '@kbn/utils'; import type { RuleAction, ThreatMapping } from '@kbn/securitysolution-io-ts-alerting-types'; import type { PartialRule } from '@kbn/alerting-plugin/server'; import type { ActionsClient } from '@kbn/actions-plugin/server'; @@ -37,8 +36,7 @@ import { createBulkErrorObject } from '../../routes/utils'; import type { RuleAlertType } from '../../rule_schema'; import { getMlRuleParams, getQueryRuleParams, getThreatRuleParams } from '../../rule_schema/mocks'; -import { createRulesAndExceptionsStreamFromNdJson } from '../logic/import/create_rules_stream_from_ndjson'; -import type { RuleExceptionsPromiseFromStreams } from '../logic/import/import_rules_utils'; +import { createPromiseFromRuleImportStream } from '../logic/import/create_promise_from_rule_import_stream'; import { internalRuleToAPIResponse } from '../logic/detection_rules_client/converters/internal_rule_to_api_response'; type PromiseFromStreams = RuleToImport | Error; @@ -50,10 +48,10 @@ const createMockImportRule = async (rule: ReturnType([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); return rules; }; @@ -433,10 +431,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); const [errors, output] = getTupleDuplicateErrorsAndUniqueRules(rules, false); const isInstanceOfError = output[0] instanceof Error; @@ -454,10 +452,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); const [errors, output] = getTupleDuplicateErrorsAndUniqueRules(rules, false); @@ -485,10 +483,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); const [errors, output] = getTupleDuplicateErrorsAndUniqueRules(rules, false); const isInstanceOfError = output[0] instanceof Error; @@ -507,10 +505,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); const [errors, output] = getTupleDuplicateErrorsAndUniqueRules(rules, true); @@ -528,10 +526,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); const [errors, output] = getTupleDuplicateErrorsAndUniqueRules(rules, false); const isInstanceOfError = output[0] instanceof Error; @@ -823,10 +821,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([]); const [errors, output] = await getInvalidConnectors(rules, clients.actionsClient); @@ -854,10 +852,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([]); const [errors, output] = await getInvalidConnectors(rules, clients.actionsClient); expect(output.length).toEqual(0); @@ -890,10 +888,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', @@ -935,10 +933,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', @@ -995,10 +993,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', @@ -1062,10 +1060,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', @@ -1131,10 +1129,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', @@ -1241,10 +1239,10 @@ describe('utils', () => { this.push(null); }, }); - const [{ rules }] = await createPromiseFromStreams([ - ndJsonStream, - ...createRulesAndExceptionsStreamFromNdJson(1000), - ]); + const [{ rules }] = await createPromiseFromRuleImportStream({ + stream: ndJsonStream, + objectLimit: 1000, + }); clients.actionsClient.getAll.mockResolvedValue([ { id: '123', From ac1a691db389b2eb5e31f6ee0fb6090017a4cdf4 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 19 Sep 2024 15:48:03 -0500 Subject: [PATCH 082/120] Cleaning up interfaces, adding tests * Removes unnecessary parameters from the `importRules` method of `DetectionRulesClient` * Adds/moves unit tests corresponding to new responsibilities * Renames RuleImportError to RuleImportErrorObject for clarity/consistency with BulkErrorObject * Adds ability to report exception list reference errors when import fails --- .../api/rules/import_rules/route.ts | 1 - ...etection_rules_client.import_rules.test.ts | 161 +++++++++++++ .../detection_rules_client.ts | 5 +- .../detection_rules_client_interface.ts | 7 +- .../methods/import_rule_with_source.ts | 4 +- .../methods/import_rules.ts | 27 ++- .../import/check_rule_exception_references.ts | 8 +- .../rule_management/logic/import/errors.ts | 10 +- .../import/import_rules_with_source.test.ts | 216 ++++++++++++------ .../logic/import/import_rules_with_source.ts | 4 - 10 files changed, 335 insertions(+), 108 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index e46a1436940d6..4a96b127df8a7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -160,7 +160,6 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C allowMissingConnectorSecrets: !!actionConnectors.length, prebuiltRulesImportHelper, detectionRulesClient, - savedObjectsClient, }); } else { importRuleResponse = await legacyImportRules({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts new file mode 100644 index 0000000000000..0d7810d74fcbb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; +import { actionsClientMock } from '@kbn/actions-plugin/server/actions_client/actions_client.mock'; +import { savedObjectsClientMock } from '@kbn/core/server/mocks'; + +import { buildMlAuthz } from '../../../../machine_learning/__mocks__/authz'; +import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; +import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; +import { prebuiltRulesImportHelperMock } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper.mock'; +import { createDetectionRulesClient } from './detection_rules_client'; +import { importRuleWithSource } from './methods/import_rule_with_source'; +import { createRuleImportErrorObject } from '../import/errors'; +import { checkRuleExceptionReferences } from '../import/check_rule_exception_references'; + +jest.mock('./methods/import_rule_with_source'); +jest.mock('../import/check_rule_exception_references'); + +describe('detectionRulesClient.importRules', () => { + let subject: ReturnType; + let ruleToImport: ReturnType; + let prebuiltRulesImportHelper: ReturnType; + + beforeEach(() => { + subject = createDetectionRulesClient({ + actionsClient: actionsClientMock.create(), + rulesClient: rulesClientMock.create(), + mlAuthz: buildMlAuthz(), + savedObjectsClient: savedObjectsClientMock.create(), + }); + + (checkRuleExceptionReferences as jest.Mock).mockReturnValue([[], []]); + (importRuleWithSource as jest.Mock).mockResolvedValue(getRulesSchemaMock()); + + ruleToImport = getImportRulesSchemaMock(); + prebuiltRulesImportHelper = prebuiltRulesImportHelperMock.create(); + prebuiltRulesImportHelper.fetchAssetRuleIds.mockResolvedValue([]); + prebuiltRulesImportHelper.fetchMatchingAssets.mockResolvedValue([]); + }); + + it('returns imported rules as RuleResponses if import was successful', async () => { + const result = await subject.importRules({ + allowMissingConnectorSecrets: false, + overwriteRules: false, + prebuiltRulesImportHelper, + rules: [ruleToImport, ruleToImport], + }); + + expect(result).toEqual([getRulesSchemaMock(), getRulesSchemaMock()]); + }); + + it('returns an import error if rule import throws an import error', async () => { + const importError = createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'an error occurred', + }); + (importRuleWithSource as jest.Mock).mockReset().mockRejectedValueOnce(importError); + + const result = await subject.importRules({ + allowMissingConnectorSecrets: false, + overwriteRules: false, + prebuiltRulesImportHelper, + rules: [ruleToImport], + }); + + expect(result).toEqual([importError]); + }); + + it('returns a generic error if rule import throws unexpectedly', async () => { + const genericError = new Error('an unexpected error occurred'); + (importRuleWithSource as jest.Mock).mockReset().mockRejectedValueOnce(genericError); + + const result = await subject.importRules({ + allowMissingConnectorSecrets: false, + overwriteRules: false, + prebuiltRulesImportHelper, + rules: [ruleToImport], + }); + + expect(result).toEqual([ + expect.objectContaining({ + error: expect.objectContaining({ + message: 'an unexpected error occurred', + ruleId: ruleToImport.rule_id, + type: 'unknown', + }), + }), + ]); + }); + + describe('when rule has no exception list references', () => { + beforeEach(() => { + (checkRuleExceptionReferences as jest.Mock).mockReset().mockReturnValueOnce([ + [ + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'list not found', + }), + ], + [], + ]); + }); + + it('returns both exception list reference errors and the imported rule if import succeeds', async () => { + const result = await subject.importRules({ + allowMissingConnectorSecrets: false, + overwriteRules: false, + prebuiltRulesImportHelper, + rules: [ruleToImport], + }); + + expect(result).toEqual([ + expect.objectContaining({ + error: expect.objectContaining({ + message: 'list not found', + ruleId: 'rule-id', + type: 'unknown', + }), + }), + getRulesSchemaMock(), + ]); + }); + + it('returns both exception list reference errors and the imported rule if import throws an error', async () => { + const importError = createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'an error occurred', + }); + (importRuleWithSource as jest.Mock).mockReset().mockRejectedValueOnce(importError); + + const result = await subject.importRules({ + allowMissingConnectorSecrets: false, + overwriteRules: false, + prebuiltRulesImportHelper, + rules: [ruleToImport], + }); + + expect(result).toEqual([ + expect.objectContaining({ + error: expect.objectContaining({ + message: 'list not found', + ruleId: 'rule-id', + type: 'unknown', + }), + }), + expect.objectContaining({ + error: expect.objectContaining({ + message: 'an error occurred', + ruleId: 'rule-id', + type: 'unknown', + }), + }), + ]); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts index 7fa2272e40fc9..81d93dac94597 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts @@ -13,7 +13,7 @@ import type { RuleResponse } from '../../../../../../common/api/detection_engine import { withSecuritySpan } from '../../../../../utils/with_security_span'; import type { MlAuthz } from '../../../../machine_learning/authz'; import { createPrebuiltRuleAssetsClient } from '../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; -import type { RuleImportError } from '../import/errors'; +import type { RuleImportErrorObject } from '../import/errors'; import type { CreateCustomRuleArgs, CreatePrebuiltRuleArgs, @@ -149,11 +149,12 @@ export const createDetectionRulesClient = ({ }); }, - async importRules(args: ImportRulesArgs): Promise> { + async importRules(args: ImportRulesArgs): Promise> { return withSecuritySpan('DetectionRulesClient.importRules', async () => { return importRules({ ...args, detectionRulesClient: this, + savedObjectsClient, }); }); }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts index de78d7fdf2665..517bebddc60ee 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts @@ -5,7 +5,6 @@ * 2.0. */ -import type { SavedObjectsClientContract } from '@kbn/core/server'; import type { RuleCreateProps, RuleUpdateProps, @@ -16,7 +15,7 @@ import type { RuleToImport, RuleSource, } from '../../../../../../common/api/detection_engine'; -import type { RuleImportError } from '../import/errors'; +import type { RuleImportErrorObject } from '../import/errors'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; @@ -29,7 +28,7 @@ export interface IDetectionRulesClient { upgradePrebuiltRule: (args: UpgradePrebuiltRuleArgs) => Promise; legacyImportRule: (args: LegacyImportRuleArgs) => Promise; importRule: (args: ImportRuleArgs) => Promise; - importRules: (args: ImportRulesArgs) => Promise>; + importRules: (args: ImportRulesArgs) => Promise>; } export interface CreateCustomRuleArgs { @@ -70,9 +69,7 @@ export interface ImportRuleArgs { export interface ImportRulesArgs { rules: RuleToImport[]; - detectionRulesClient: IDetectionRulesClient; overwriteRules: boolean; prebuiltRulesImportHelper: PrebuiltRulesImportHelper; allowMissingConnectorSecrets?: boolean; - savedObjectsClient: SavedObjectsClientContract; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts index 1ac02ee63c0b2..408485b2952dc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts @@ -20,7 +20,7 @@ import { applyRuleDefaults } from '../mergers/apply_rule_defaults'; import { validateMlAuth, toggleRuleEnabledOnUpdate } from '../utils'; import { getRuleByRuleId } from './get_rule_by_rule_id'; import { SERVER_APP_ID } from '../../../../../../../common'; -import { createRuleImportError } from '../../import/errors'; +import { createRuleImportErrorObject } from '../../import/errors'; interface ImportRuleOptions { actionsClient: ActionsClient; @@ -53,7 +53,7 @@ export const importRuleWithSource = async ({ }); if (existingRule && !overwriteRules) { - throw createRuleImportError({ + throw createRuleImportErrorObject({ ruleId: existingRule.rule_id, type: 'conflict', message: `rule_id: "${existingRule.rule_id}" already exists`, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts index 0dcc8912b996f..ead7456d5b9b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts @@ -12,8 +12,8 @@ import type { RuleResponse, RuleToImport } from '../../../../../../../common/api import { ruleToImportHasVersion } from '../../../../../../../common/api/detection_engine/rule_management'; import type { PrebuiltRulesImportHelper } from '../../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; import { - type RuleImportError, - createRuleImportError, + type RuleImportErrorObject, + createRuleImportErrorObject, isRuleImportError, } from '../../import/errors'; import { checkRuleExceptionReferences } from '../../import/check_rule_exception_references'; @@ -39,7 +39,7 @@ export const importRules = async ({ prebuiltRulesImportHelper: PrebuiltRulesImportHelper; rules: RuleToImport[]; savedObjectsClient: SavedObjectsClientContract; -}): Promise> => { +}): Promise> => { const existingLists = await getReferencedExceptionLists({ rules, savedObjectsClient, @@ -55,7 +55,7 @@ export const importRules = async ({ return Promise.all( rules.map(async (rule) => { if (!ruleToImportHasVersion(rule)) { - return createRuleImportError({ + return createRuleImportErrorObject({ message: i18n.translate( 'xpack.securitySolution.detectionEngine.rules.cannotImportRuleWithoutVersion', { @@ -67,6 +67,8 @@ export const importRules = async ({ }); } + const errors: RuleImportErrorObject[] = []; + const { immutable, ruleSource } = calculateRuleSourceForImport({ rule, prebuiltRuleAssets, @@ -78,6 +80,7 @@ export const importRules = async ({ rule, existingLists, }); + errors.push(...exceptionErrors); const importedRule = await detectionRulesClient.importRule({ ruleToImport: { @@ -90,18 +93,18 @@ export const importRules = async ({ allowMissingConnectorSecrets, }); - return [...exceptionErrors, importedRule]; + return [...errors, importedRule]; } catch (err) { const { error, message } = err; - if (isRuleImportError(err)) { - return err; - } + const caughtError = isRuleImportError(err) + ? err + : createRuleImportErrorObject({ + ruleId: rule.rule_id, + message: message ?? error?.message ?? 'unknown error', + }); - return createRuleImportError({ - ruleId: rule.rule_id, - message: message ?? error?.message ?? 'unknown error', - }); + return [...errors, caughtError]; } }) ).then((results) => results.flat()); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts index 9e54cc1ee816a..2d89fbf956536 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/check_rule_exception_references.ts @@ -7,7 +7,7 @@ import type { ListArray, ExceptionListSchema } from '@kbn/securitysolution-io-ts-list-types'; import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; -import { type RuleImportError, createRuleImportError } from './errors'; +import { type RuleImportErrorObject, createRuleImportErrorObject } from './errors'; /** * Helper to check if all the exception lists referenced on a @@ -26,9 +26,9 @@ export const checkRuleExceptionReferences = ({ }: { rule: RuleToImport; existingLists: Record; -}): [RuleImportError[], ListArray] => { +}): [RuleImportErrorObject[], ListArray] => { let ruleExceptions: ListArray = []; - let errors: RuleImportError[] = []; + let errors: RuleImportErrorObject[] = []; const { rule_id: ruleId } = rule; const exceptionLists = rule.exceptions_list ?? []; @@ -53,7 +53,7 @@ export const checkRuleExceptionReferences = ({ // this error to notify a user of the action taken. errors = [ ...errors, - createRuleImportError({ + createRuleImportErrorObject({ ruleId, message: `Rule with rule_id: "${ruleId}" references a non existent exception list of list_id: "${exceptionList.list_id}". Reference has been removed.`, }), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts index 2d81aa56a138a..77945aa8a3fa5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/errors.ts @@ -15,7 +15,7 @@ export type RuleImportErrorType = 'conflict' | 'unknown'; * * NOTE that this does not inherit from Error */ -export interface RuleImportError { +export interface RuleImportErrorObject { error: { ruleId: string; message: string; @@ -23,7 +23,7 @@ export interface RuleImportError { }; } -export const createRuleImportError = ({ +export const createRuleImportErrorObject = ({ ruleId, message, type, @@ -31,7 +31,7 @@ export const createRuleImportError = ({ ruleId: string; message: string; type?: RuleImportErrorType; -}): RuleImportError => ({ +}): RuleImportErrorObject => ({ error: { ruleId, message, @@ -39,11 +39,11 @@ export const createRuleImportError = ({ }, }); -export const isRuleImportError = (obj: unknown): obj is RuleImportError => +export const isRuleImportError = (obj: unknown): obj is RuleImportErrorObject => has(obj, 'error') && has(obj, 'error.ruleId') && has(obj, 'error.type') && has(obj, 'error.message'); -export const isRuleConflictError = (error: RuleImportError): boolean => +export const isRuleConflictError = (error: RuleImportErrorObject): boolean => error.error.type === 'conflict'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts index 9474eee0b189d..88742354d9abc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts @@ -5,30 +5,30 @@ * 2.0. */ -import type { SavedObjectsClientContract } from '@kbn/core/server'; -import { savedObjectsClientMock } from '@kbn/core/server/mocks'; -import { getValidatedRuleToImportMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; +import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; -import { requestContextMock } from '../../../routes/__mocks__'; import { prebuiltRulesImportHelperMock } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper.mock'; import { importRules } from './import_rules_with_source'; -import { createBulkErrorObject } from '../../../routes/utils'; +import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; +import { detectionRulesClientMock } from '../detection_rules_client/__mocks__/detection_rules_client'; +import { createRuleImportErrorObject } from './errors'; describe('importRules', () => { - const { clients, context } = requestContextMock.createTools(); - const ruleToImport = getValidatedRuleToImportMock(); + let ruleToImport: ReturnType; - let savedObjectsClient: jest.Mocked; - let mockPrebuiltRulesImportHelper: ReturnType; + let detectionRulesClient: jest.Mocked; + let prebuiltRulesImportHelper: ReturnType; beforeEach(() => { jest.clearAllMocks(); - savedObjectsClient = savedObjectsClientMock.create(); - mockPrebuiltRulesImportHelper = prebuiltRulesImportHelperMock.create(); - mockPrebuiltRulesImportHelper.fetchMatchingAssets.mockResolvedValue([]); - mockPrebuiltRulesImportHelper.fetchAssetRuleIds.mockResolvedValue([]); + detectionRulesClient = detectionRulesClientMock.create(); + detectionRulesClient.importRules.mockResolvedValue([]); + ruleToImport = getImportRulesSchemaMock(); + prebuiltRulesImportHelper = prebuiltRulesImportHelperMock.create(); + prebuiltRulesImportHelper.fetchMatchingAssets.mockResolvedValue([]); + prebuiltRulesImportHelper.fetchAssetRuleIds.mockResolvedValue([]); }); it('returns an empty rules response if no rules to import', async () => { @@ -36,28 +36,35 @@ describe('importRules', () => { ruleChunks: [], rulesResponseAcc: [], overwriteRules: false, - detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - savedObjectsClient, - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, + detectionRulesClient, + prebuiltRulesImportHelper, }); expect(result).toEqual([]); }); - it('returns 400 error if "ruleChunks" includes Error', async () => { + it('returns 400 errors if import was attempted on errored rules', async () => { + const rules = [new Error('error parsing'), new Error('error parsing')]; + const result = await importRules({ - ruleChunks: [[new Error('error importing')]], + ruleChunks: [rules], rulesResponseAcc: [], overwriteRules: false, - detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, - savedObjectsClient, + detectionRulesClient, + prebuiltRulesImportHelper, }); expect(result).toEqual([ { error: { - message: 'error importing', + message: 'error parsing', + status_code: 400, + }, + rule_id: '(unknown id)', + }, + { + error: { + message: 'error parsing', status_code: 400, }, rule_id: '(unknown id)', @@ -65,83 +72,146 @@ describe('importRules', () => { ]); }); - it('returns 409 error if DetectionRulesClient throws with 409 - existing rule', async () => { - clients.detectionRulesClient.importRule.mockImplementationOnce(async () => { - throw createBulkErrorObject({ - ruleId: ruleToImport.rule_id, - statusCode: 409, - message: `rule_id: "${ruleToImport.rule_id}" already exists`, - }); + it('returns 400 errors if client import returns generic errors', async () => { + detectionRulesClient.importRules.mockResolvedValueOnce([ + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'import error', + }), + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'import error', + }), + ]); + + const result = await importRules({ + ruleChunks: [[ruleToImport]], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient, + prebuiltRulesImportHelper, }); - const ruleChunk = [ruleToImport]; + expect(result).toEqual([ + { + error: { + message: 'import error', + status_code: 400, + }, + rule_id: 'rule-id', + }, + { + error: { + message: 'import error', + status_code: 400, + }, + rule_id: 'rule-id', + }, + ]); + }); + + it('returns 409 errors if client import returns conflict errors', async () => { + detectionRulesClient.importRules.mockResolvedValueOnce([ + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'import conflict', + type: 'conflict', + }), + createRuleImportErrorObject({ + ruleId: 'rule-id-2', + message: 'import conflict', + type: 'conflict', + }), + ]); + const result = await importRules({ - ruleChunks: [ruleChunk], + ruleChunks: [[ruleToImport]], rulesResponseAcc: [], overwriteRules: false, - detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, - savedObjectsClient, + detectionRulesClient, + prebuiltRulesImportHelper, }); expect(result).toEqual([ { error: { - message: `rule_id: "${ruleToImport.rule_id}" already exists`, + message: 'import conflict', status_code: 409, }, - rule_id: ruleToImport.rule_id, + rule_id: 'rule-id', + }, + { + error: { + message: 'import conflict', + status_code: 409, + }, + rule_id: 'rule-id-2', }, ]); }); - it('creates rule if no matching existing rule found', async () => { - clients.detectionRulesClient.importRule.mockResolvedValue({ - ...getRulesSchemaMock(), - rule_id: ruleToImport.rule_id, - }); + it('returns a combination of 200s and 4xxs if some rules were imported and some errored', async () => { + detectionRulesClient.importRules.mockResolvedValueOnce([ + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'parse error', + }), + getRulesSchemaMock(), + createRuleImportErrorObject({ + ruleId: 'rule-id-2', + message: 'import conflict', + type: 'conflict', + }), + getRulesSchemaMock(), + ]); + const successfulRuleId = getRulesSchemaMock().rule_id; - const ruleChunk = [ruleToImport]; const result = await importRules({ - ruleChunks: [ruleChunk], + ruleChunks: [[ruleToImport]], rulesResponseAcc: [], overwriteRules: false, - detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, - savedObjectsClient, + detectionRulesClient, + prebuiltRulesImportHelper, }); - expect(result).toEqual([{ rule_id: ruleToImport.rule_id, status_code: 200 }]); + expect(result).toEqual([ + { + error: { + message: 'parse error', + status_code: 400, + }, + rule_id: 'rule-id', + }, + { rule_id: successfulRuleId, status_code: 200 }, + { + error: { + message: 'import conflict', + status_code: 409, + }, + rule_id: 'rule-id-2', + }, + { rule_id: successfulRuleId, status_code: 200 }, + ]); }); - describe('compatibility with prebuilt rules', () => { - let prebuiltRuleToImport: ReturnType; + it('returns 200s if all rules were imported successfully', async () => { + detectionRulesClient.importRules.mockResolvedValueOnce([ + getRulesSchemaMock(), + getRulesSchemaMock(), + ]); + const successfulRuleId = getRulesSchemaMock().rule_id; - beforeEach(() => { - prebuiltRuleToImport = { - ...getValidatedRuleToImportMock(), - immutable: true, - version: 1, - }; + const result = await importRules({ + ruleChunks: [[ruleToImport]], + rulesResponseAcc: [], + overwriteRules: false, + detectionRulesClient, + prebuiltRulesImportHelper, }); - it('imports a prebuilt rule', async () => { - clients.detectionRulesClient.importRule.mockResolvedValue({ - ...getRulesSchemaMock(), - rule_id: prebuiltRuleToImport.rule_id, - }); - - const ruleChunk = [prebuiltRuleToImport]; - const result = await importRules({ - ruleChunks: [ruleChunk], - rulesResponseAcc: [], - overwriteRules: false, - detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - prebuiltRulesImportHelper: mockPrebuiltRulesImportHelper, - savedObjectsClient, - }); - - expect(result).toEqual([{ rule_id: prebuiltRuleToImport.rule_id, status_code: 200 }]); - }); + expect(result).toEqual([ + { rule_id: successfulRuleId, status_code: 200 }, + { rule_id: successfulRuleId, status_code: 200 }, + ]); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts index f8cc2431466e0..a8d7e6516d6cb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts @@ -36,7 +36,6 @@ export const importRules = async ({ detectionRulesClient, prebuiltRulesImportHelper, allowMissingConnectorSecrets, - savedObjectsClient, }: { ruleChunks: RuleFromImportStream[][]; rulesResponseAcc: ImportRuleResponse[]; @@ -44,7 +43,6 @@ export const importRules = async ({ detectionRulesClient: IDetectionRulesClient; prebuiltRulesImportHelper: PrebuiltRulesImportHelper; allowMissingConnectorSecrets?: boolean; - savedObjectsClient: SavedObjectsClientContract; }) => { let response: ImportRuleResponse[] = [...rulesResponseAcc]; @@ -62,11 +60,9 @@ export const importRules = async ({ const importedRulesResponse = await detectionRulesClient.importRules({ allowMissingConnectorSecrets, - detectionRulesClient, overwriteRules, prebuiltRulesImportHelper, rules, - savedObjectsClient, }); const genericErrors = errors.map((error) => From 8f1015a7bfe53d1bf7799f91ab6c0572284d0909 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 19 Sep 2024 22:40:35 -0500 Subject: [PATCH 083/120] PrebuiltRulesImportHelper -> RuleSourceImporter This is a significant refactor of the responsibilities of The Class Formerly Known as PrebuiltRulesImportHelper. Besides the (hopefully) more descriptive name (as the class is responsible with generating appropriate `rule_source` values during import), the class now combines what were previously multiple sequential method calls into a single one (still `setup()`), while adding some additional methods to aid in `rule_source` calculation. The big one (and the one that incited this change) was the ability to ask of an incoming (importing) rule: does this represent a prebuilt rule, or not? This is self-evident from a calculated `rule_source` (i.e. `rule_source.type === 'internal'`), but for this use case (defaulting a prebuilt rule's version) we need to do so _before_ calculating the full `rule_source`. In order to make this determination, we need our list of rule assets, which were already fetched by the instance. But rather than require the caller to take those assets and pass them to another helper function, I opted instead to move that logic to a method on the RuleSourceImporter. Similarly, I replaced the manual invocation of `calculateRuleSourceForImport` with a (wrapping) method on the RuleSourceImporter. --- .../prebuilt_rules_import_helper.mock.ts | 19 -- .../prebuilt_rules_import_helper.test.ts | 166 ------------------ .../logic/prebuilt_rules_import_helper.ts | 135 -------------- .../logic/rule_source_importer/index.ts | 9 + .../rule_source_importer.mock.ts | 19 ++ .../rule_source_importer.test.ts | 150 ++++++++++++++++ .../rule_source_importer.ts | 166 ++++++++++++++++++ .../logic/rule_source_importer/types.ts | 28 +++ .../logic/rule_source_importer/utils.ts | 65 +++++++ .../api/rules/import_rules/route.ts | 9 +- ...etection_rules_client.import_rules.test.ts | 22 +-- .../detection_rules_client_interface.ts | 4 +- .../methods/import_rules.ts | 54 +++--- .../create_promise_from_rule_import_stream.ts | 2 +- .../logic/import/import_rules_utils.ts | 2 +- .../import/import_rules_with_source.test.ts | 20 +-- .../logic/import/import_rules_with_source.ts | 16 +- .../logic/import/{types.ts => utils.ts} | 3 + 18 files changed, 500 insertions(+), 389 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.mock.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.test.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/types.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts rename x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/{types.ts => utils.ts} (78%) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.mock.ts deleted file mode 100644 index 9b14987d82519..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.mock.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { PrebuiltRulesImportHelper } from './prebuilt_rules_import_helper'; - -const createPrebuiltRulesImportHelperMock = (): jest.Mocked => - ({ - setup: jest.fn(), - fetchMatchingAssets: jest.fn(), - fetchAssetRuleIds: jest.fn(), - } as unknown as jest.Mocked); - -export const prebuiltRulesImportHelperMock = { - create: createPrebuiltRulesImportHelperMock, -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.test.ts deleted file mode 100644 index 515b880079acc..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { SavedObjectsClientContract } from '@kbn/core/server'; -import { savedObjectsClientMock } from '@kbn/core/server/mocks'; -import type { RuleToImport } from '../../../../../common/api/detection_engine'; -import { createPrebuiltRuleAssetsClient as createPrebuiltRuleAssetsClientMock } from './rule_assets/__mocks__/prebuilt_rule_assets_client'; -import { ensureLatestRulesPackageInstalled } from './ensure_latest_rules_package_installed'; -import { configMock, createMockConfig, requestContextMock } from '../../routes/__mocks__'; -import { PrebuiltRulesImportHelper } from './prebuilt_rules_import_helper'; - -jest.mock('./ensure_latest_rules_package_installed'); - -let mockPrebuiltRuleAssetsClient: ReturnType; - -jest.mock('./rule_assets/prebuilt_rule_assets_client', () => ({ - createPrebuiltRuleAssetsClient: () => mockPrebuiltRuleAssetsClient, -})); - -describe('PrebuiltRulesImportHelper', () => { - let config: ReturnType; - let context: ReturnType['securitySolution']; - let savedObjectsClient: jest.Mocked; - - beforeEach(() => { - config = createMockConfig(); - config = configMock.withExperimentalFeature(config, 'prebuiltRulesCustomizationEnabled'); - context = requestContextMock.create().securitySolution; - savedObjectsClient = savedObjectsClientMock.create(); - mockPrebuiltRuleAssetsClient = createPrebuiltRuleAssetsClientMock(); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - it('should initialize correctly', () => { - const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - - expect(importer).toBeDefined(); - expect(importer.enabled).toBe(true); - expect(importer.latestPackagesInstalled).toBe(false); - }); - - describe('setup', () => { - it('should not call ensureLatestRulesPackageInstalled if disabled', async () => { - config = createMockConfig(); - const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - - await importer.setup(); - - expect(ensureLatestRulesPackageInstalled).not.toHaveBeenCalled(); - expect(importer.latestPackagesInstalled).toBe(false); - }); - - it('should call ensureLatestRulesPackageInstalled if enabled', async () => { - const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - - await importer.setup(); - - expect(ensureLatestRulesPackageInstalled).toHaveBeenCalledWith( - mockPrebuiltRuleAssetsClient, - config, - context - ); - expect(importer.latestPackagesInstalled).toBe(true); - }); - }); - - describe('fetchMatchingAssets', () => { - it('should return an empty array if disabled', async () => { - config = createMockConfig(); - const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - - const result = await importer.fetchMatchingAssets({ rules: [] }); - - expect(result).toEqual([]); - }); - - it('should throw an error if latestPackagesInstalled is false', async () => { - const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - - await expect(importer.fetchMatchingAssets({ rules: [] })).rejects.toThrow( - 'Prebuilt rule assets cannot be fetched until the latest rules package is installed. Call setup() on this object first.' - ); - }); - - it('should fetch prebuilt rule assets correctly if latestPackagesInstalled is true', async () => { - const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - importer.latestPackagesInstalled = true; - - const rules = [ - { rule_id: 'rule-1', version: 1 }, - { rule_id: 'rule-2', version: 2 }, - new Error('Invalid rule'), - ] as Array; - - await importer.fetchMatchingAssets({ rules }); - - expect(mockPrebuiltRuleAssetsClient.fetchAssetsByVersion).toHaveBeenCalledWith([ - { rule_id: 'rule-1', version: 1 }, - { rule_id: 'rule-2', version: 2 }, - ]); - }); - }); - - describe('fetchAssetRuleIds', () => { - it('returns an empty array if the importer is not enabled', async () => { - const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - importer.enabled = false; - - const result = await importer.fetchAssetRuleIds({ rules: [] }); - - expect(result).toEqual([]); - }); - - it('throws an error if the latest packages are not installed', async () => { - const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - importer.latestPackagesInstalled = false; - - await expect(importer.fetchAssetRuleIds({ rules: [] })).rejects.toThrow( - 'Installed rule IDs cannot be fetched until the latest rules package is installed. Call setup() on this object first.' - ); - }); - - it('fetches and return the rule IDs of installed prebuilt rules', async () => { - const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - importer.latestPackagesInstalled = true; - const rules = [ - { rule_id: 'rule-1', version: 1 }, - { rule_id: 'rule-2', version: 1 }, - new Error('Invalid rule'), - ] as Array; - const installedRuleAssets = [{ rule_id: 'rule-1' }, { rule_id: 'rule-2' }]; - - (mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId as jest.Mock).mockResolvedValue( - installedRuleAssets - ); - - const result = await importer.fetchAssetRuleIds({ rules }); - - expect(mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId).toHaveBeenCalledWith([ - 'rule-1', - 'rule-2', - ]); - expect(result).toEqual(['rule-1', 'rule-2']); - }); - - it('handles rules that are instances of Error', async () => { - const importer = new PrebuiltRulesImportHelper({ config, context, savedObjectsClient }); - importer.latestPackagesInstalled = true; - const rules = [new Error('Invalid rule')]; - - (mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId as jest.Mock).mockResolvedValue([]); - - const result = await importer.fetchAssetRuleIds({ rules }); - - expect(mockPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId).toHaveBeenCalledWith([]); - expect(result).toEqual([]); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.ts deleted file mode 100644 index 79ce18f407de7..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/prebuilt_rules_import_helper.ts +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { SavedObjectsClientContract } from '@kbn/core/server'; -import type { RuleToImport } from '../../../../../common/api/detection_engine'; -import type { SecuritySolutionApiRequestHandlerContext } from '../../../../types'; -import type { ConfigType } from '../../../../config'; -import type { PrebuiltRuleAsset } from '../model/rule_assets/prebuilt_rule_asset'; -import { createPrebuiltRuleAssetsClient } from './rule_assets/prebuilt_rule_assets_client'; -import { ensureLatestRulesPackageInstalled } from './ensure_latest_rules_package_installed'; - -type MaybeRule = RuleToImport | Error; - -export interface IPrebuiltRulesImportHelper { - setup: () => Promise; - fetchMatchingAssets: ({ rules }: { rules: MaybeRule[] }) => Promise; - fetchAssetRuleIds: ({ rules }: { rules: MaybeRule[] }) => Promise; -} - -/** - * - * This class contains utilities for assisting with the importing of prebuilt - * rules. It ensures that the system contains the necessary assets, and also - * provides utilities for fetching information from them, necessary for - * importing prebuilt rules. - */ -export class PrebuiltRulesImportHelper implements IPrebuiltRulesImportHelper { - private context: SecuritySolutionApiRequestHandlerContext; - private config: ConfigType; - private ruleAssetsClient: ReturnType; - public enabled: boolean; - public latestPackagesInstalled: boolean = false; - - constructor({ - config, - savedObjectsClient, - context, - }: { - config: ConfigType; - context: SecuritySolutionApiRequestHandlerContext; - savedObjectsClient: SavedObjectsClientContract; - }) { - this.ruleAssetsClient = createPrebuiltRuleAssetsClient(savedObjectsClient); - this.context = context; - this.config = config; - this.enabled = config.experimentalFeatures.prebuiltRulesCustomizationEnabled; - } - - /** - * - * Prepares the system to import prebuilt rules by ensuring the latest rules - * package is installed. - */ - public async setup() { - if (!this.enabled) { - return; - } - - await ensureLatestRulesPackageInstalled(this.ruleAssetsClient, this.config, this.context); - this.latestPackagesInstalled = true; - } - - /** - * Retrieves prebuilt rule assets for rules that are being imported. - * @param rules - The rules being imported - * - * @returns The prebuilt rule assets corresponding to the specified prebuilt - * rules, which are used to determine how to import those rules (create vs. update, etc.). - * Assets match the `rule_id` and `version` of the specified rules. - */ - public async fetchMatchingAssets({ - rules, - }: { - rules: MaybeRule[]; - }): Promise { - if (!this.enabled) { - return []; - } - - if (!this.latestPackagesInstalled) { - throw new Error( - 'Prebuilt rule assets cannot be fetched until the latest rules package is installed. Call setup() on this object first.' - ); - } - - const prebuiltRulesToImport = rules.flatMap((rule) => { - if (rule instanceof Error || rule.version == null) { - return []; - } - return { - rule_id: rule.rule_id, - version: rule.version, - }; - }); - - return this.ruleAssetsClient.fetchAssetsByVersion(prebuiltRulesToImport); - } - - /** - * Retrieves the rule IDs (`rule_id`s) of available prebuilt rule assets matching those - * of the specified rules. This information can be used to determine whether - * the rule being imported is a custom rule or a prebuilt rule. - * - * @param rules - A list of rules being imported. - * - * @returns A list of the prebuilt rule IDs that are available. - * - */ - public async fetchAssetRuleIds({ rules }: { rules: MaybeRule[] }): Promise { - if (!this.enabled) { - return []; - } - - if (!this.latestPackagesInstalled) { - throw new Error( - 'Installed rule IDs cannot be fetched until the latest rules package is installed. Call setup() on this object first.' - ); - } - - const ruleIds = rules.flatMap((rule) => { - if (rule instanceof Error) { - return []; - } - return rule.rule_id; - }); - - const installedRuleAssets = await this.ruleAssetsClient.fetchLatestAssetsByRuleId(ruleIds); - - return installedRuleAssets.map((asset) => asset.rule_id); - } -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/index.ts new file mode 100644 index 0000000000000..91087c6b607a1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './types'; +export * from './rule_source_importer'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock.ts new file mode 100644 index 0000000000000..983043e904c60 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleSourceImporter } from './rule_source_importer'; + +const createRuleSourceImporterMock = (): jest.Mocked => + ({ + setup: jest.fn(), + calculateRuleSource: jest.fn(), + isPrebuiltRule: jest.fn(), + } as unknown as jest.Mocked); + +export const ruleSourceImporterMock = { + create: createRuleSourceImporterMock, +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts new file mode 100644 index 0000000000000..3ad25a526c464 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ValidatedRuleToImport } from '../../../../../../common/api/detection_engine'; +import { createPrebuiltRuleAssetsClient as createPrebuiltRuleAssetsClientMock } from '../rule_assets/__mocks__/prebuilt_rule_assets_client'; +import type { RuleFromImportStream } from '../../../rule_management/logic/import/utils'; +import { configMock, createMockConfig, requestContextMock } from '../../../routes/__mocks__'; +import { createRuleSourceImporter } from './rule_source_importer'; +import * as calculateRuleSourceModule from '../../../rule_management/logic/import/calculate_rule_source_for_import'; +import { getPrebuiltRuleMock } from '../../mocks'; + +describe('ruleSourceImporter', () => { + let ruleAssetsClientMock: ReturnType; + let config: ReturnType; + let context: ReturnType['securitySolution']; + let ruleToImport: RuleFromImportStream; + let subject: ReturnType; + + beforeEach(() => { + config = createMockConfig(); + config = configMock.withExperimentalFeature(config, 'prebuiltRulesCustomizationEnabled'); + context = requestContextMock.create().securitySolution; + ruleAssetsClientMock = createPrebuiltRuleAssetsClientMock(); + ruleAssetsClientMock.fetchLatestAssets.mockResolvedValue([{}]); + ruleToImport = { rule_id: 'rule-1', version: 1 } as RuleFromImportStream; + + subject = createRuleSourceImporter({ + context, + config, + prebuiltRuleAssetsClient: ruleAssetsClientMock, + }); + }); + + it('should initialize correctly', () => { + expect(subject).toBeDefined(); + + expect(() => subject.isPrebuiltRule(ruleToImport)).toThrowErrorMatchingInlineSnapshot( + `"Rule rule-1 was not registered during setup."` + ); + }); + + describe('#setup()', () => { + it('does nothing if feature flag is disabled', async () => { + const disabledSubject = createRuleSourceImporter({ + config: createMockConfig(), + context, + prebuiltRuleAssetsClient: ruleAssetsClientMock, + }); + await disabledSubject.setup({ rules: [ruleToImport] }); + + expect(ruleAssetsClientMock.fetchLatestAssets).toHaveBeenCalledTimes(0); + }); + + it('fetches the rules package on the initial call', async () => { + await subject.setup({ rules: [] }); + + expect(ruleAssetsClientMock.fetchLatestAssets).toHaveBeenCalledTimes(1); + }); + + it('does not fetch the rules package on subsequent calls', async () => { + await subject.setup({ rules: [] }); + await subject.setup({ rules: [] }); + await subject.setup({ rules: [] }); + + expect(ruleAssetsClientMock.fetchLatestAssets).toHaveBeenCalledTimes(1); + }); + }); + + describe('#isPrebuiltRule()', () => { + beforeEach(() => {}); + + it("returns false if the rule's rule_id doesn't match an available rule asset", async () => { + ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([]); + await subject.setup({ rules: [ruleToImport] }); + + expect(subject.isPrebuiltRule(ruleToImport)).toBe(false); + }); + + it("returns true if the rule's rule_id matches an available rule asset", async () => { + ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([ruleToImport]); + await subject.setup({ rules: [ruleToImport] }); + + expect(subject.isPrebuiltRule(ruleToImport)).toBe(true); + }); + + it('throws an error if the rule is not known to the calculator', async () => { + ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([ruleToImport]); + await subject.setup({ rules: [ruleToImport] }); + + expect(() => + subject.isPrebuiltRule({ rule_id: 'other-rule' } as RuleFromImportStream) + ).toThrowErrorMatchingInlineSnapshot(`"Rule other-rule was not registered during setup."`); + }); + + it('throws an error if the calculator is not set up', () => { + expect(() => subject.isPrebuiltRule(ruleToImport)).toThrowErrorMatchingInlineSnapshot( + `"Rule rule-1 was not registered during setup."` + ); + }); + }); + + describe('#calculateRuleSource()', () => { + let rule: ValidatedRuleToImport; + + beforeEach(() => { + rule = { rule_id: 'validated-rule', version: 1 } as ValidatedRuleToImport; + }); + + it('invokes calculateRuleSourceForImport with the correct arguments', async () => { + ruleAssetsClientMock.fetchAssetsByVersion.mockResolvedValue([ + getPrebuiltRuleMock({ rule_id: 'rule-1' }), + ]); + ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([ + getPrebuiltRuleMock({ rule_id: 'rule-2' }), + ]); + const calculatorSpy = jest + .spyOn(calculateRuleSourceModule, 'calculateRuleSourceForImport') + .mockReturnValue({ ruleSource: { type: 'internal' }, immutable: false }); + + await subject.setup({ rules: [rule] }); + await subject.calculateRuleSource(rule); + + expect(calculatorSpy).toHaveBeenCalledTimes(1); + expect(calculatorSpy).toHaveBeenCalledWith({ + rule, + prebuiltRuleAssets: [expect.objectContaining({ rule_id: 'rule-1' })], + installedRuleIds: ['rule-2'], + }); + }); + + it('throws an error if the rule is not known to the calculator', async () => { + ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([ruleToImport]); + await subject.setup({ rules: [ruleToImport] }); + + expect(() => subject.calculateRuleSource(rule)).toThrowErrorMatchingInlineSnapshot( + `"Rule validated-rule was not registered during setup."` + ); + }); + + it('throws an error if the calculator is not set up', async () => { + expect(() => subject.calculateRuleSource(rule)).toThrowErrorMatchingInlineSnapshot( + `"Rule validated-rule was not registered during setup."` + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts new file mode 100644 index 0000000000000..fcdf27b472d40 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SecuritySolutionApiRequestHandlerContext } from '../../../../../types'; +import type { ConfigType } from '../../../../../config'; +import type { + RuleToImport, + ValidatedRuleToImport, +} from '../../../../../../common/api/detection_engine'; +import { calculateRuleSourceForImport } from '../../../rule_management/logic/import/calculate_rule_source_for_import'; +import { + isRuleToImport, + type RuleFromImportStream, +} from '../../../rule_management/logic/import/utils'; +import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; +import type { IPrebuiltRuleAssetsClient } from '../rule_assets/prebuilt_rule_assets_client'; +import { ensureLatestRulesPackageInstalled } from '../ensure_latest_rules_package_installed'; +import type { CalculatedRuleSource, IRuleSourceImporter, RuleSpecifier } from './types'; +import { fetchAvailableRuleAssetIds, fetchMatchingAssets } from './utils'; + +/** + * + * This class contains utilities for assisting with the calculation of + * `rule_source` during import. It ensures that the system contains the + * necessary assets, and provides utilities for fetching information from them, + * necessary for said calculation. + */ +export class RuleSourceImporter implements IRuleSourceImporter { + private context: SecuritySolutionApiRequestHandlerContext; + private config: ConfigType; + private ruleAssetsClient: IPrebuiltRuleAssetsClient; + private enabled: boolean; + private latestPackagesInstalled: boolean = false; + private matchingAssets: PrebuiltRuleAsset[] = []; + private knownRules: RuleSpecifier[] = []; + private availableRuleAssetIds: string[] = []; + + constructor({ + config, + context, + prebuiltRuleAssetsClient, + }: { + config: ConfigType; + context: SecuritySolutionApiRequestHandlerContext; + prebuiltRuleAssetsClient: IPrebuiltRuleAssetsClient; + }) { + this.ruleAssetsClient = prebuiltRuleAssetsClient; + this.context = context; + this.config = config; + this.enabled = config.experimentalFeatures.prebuiltRulesCustomizationEnabled; + } + + /** + * + * Prepares the importing of rules by ensuring the latest rules + * package is installed and fetching the associated prebuilt rule assets. + */ + public async setup({ rules }: { rules: RuleFromImportStream[] }): Promise { + if (!this.enabled) { + return; + } + + if (!this.latestPackagesInstalled) { + await ensureLatestRulesPackageInstalled(this.ruleAssetsClient, this.config, this.context); + this.latestPackagesInstalled = true; + } + + this.knownRules = rules + .filter(isRuleToImport) + .map((rule) => ({ rule_id: rule.rule_id, version: rule.version })); + + try { + this.validateSetupState(); + + this.matchingAssets = await this.fetchMatchingAssets(); + this.availableRuleAssetIds = await this.fetchAvailableRuleAssetIds(); + } catch (e) { + // If all incoming rules were errors, there's nothing to calculate + } + } + + public isPrebuiltRule(rule: RuleFromImportStream): boolean { + if (!isRuleToImport(rule)) { + return false; + } + this.validateRuleInput(rule); + + return this.availableRuleAssetIds.includes(rule.rule_id); + } + + public calculateRuleSource(rule: ValidatedRuleToImport): CalculatedRuleSource { + this.validateRuleInput(rule); + + return calculateRuleSourceForImport({ + rule, + prebuiltRuleAssets: this.matchingAssets, + installedRuleIds: this.availableRuleAssetIds, + }); + } + + private async fetchMatchingAssets(): Promise { + this.validateSetupState(); + + return fetchMatchingAssets({ + rules: this.knownRules, + ruleAssetsClient: this.ruleAssetsClient, + }); + } + + private async fetchAvailableRuleAssetIds(): Promise { + this.validateSetupState(); + + return fetchAvailableRuleAssetIds({ + rules: this.knownRules, + ruleAssetsClient: this.ruleAssetsClient, + }); + } + + /** + * Runtime sanity checks to ensure no one's calling this stateful instance in the wrong way. + * */ + private validateSetupState() { + if (!this.latestPackagesInstalled) { + throw new Error('Expected rules package to be installed'); + } + + if (this.knownRules.length === 0) { + throw new Error( + 'Expected rules to have been registered, but none were. Please call the setup() method with the relevant incoming rules.' + ); + } + } + + private validateRuleInput(rule: RuleToImport) { + if ( + !this.knownRules.some( + (knownRule) => knownRule.rule_id === rule.rule_id && knownRule.version === rule.version + ) + ) { + throw new Error(`Rule ${rule.rule_id} was not registered during setup.`); + } + } +} + +export const createRuleSourceImporter = ({ + config, + context, + prebuiltRuleAssetsClient, +}: { + config: ConfigType; + context: SecuritySolutionApiRequestHandlerContext; + prebuiltRuleAssetsClient: IPrebuiltRuleAssetsClient; +}): RuleSourceImporter => { + return new RuleSourceImporter({ config, context, prebuiltRuleAssetsClient }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/types.ts new file mode 100644 index 0000000000000..f588b36ca2b74 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/types.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + RuleSource, + ValidatedRuleToImport, +} from '../../../../../../common/api/detection_engine'; +import type { RuleFromImportStream } from '../../../rule_management/logic/import/utils'; + +export interface RuleSpecifier { + rule_id: string; + version: number | undefined; +} + +export interface CalculatedRuleSource { + ruleSource: RuleSource; + immutable: boolean; +} + +export interface IRuleSourceImporter { + setup: ({ rules }: { rules: RuleFromImportStream[] }) => Promise; + isPrebuiltRule: (rule: RuleFromImportStream) => boolean; + calculateRuleSource: (rule: ValidatedRuleToImport) => CalculatedRuleSource; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts new file mode 100644 index 0000000000000..41d3a71825f2f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; +import type { IPrebuiltRuleAssetsClient } from '../rule_assets/prebuilt_rule_assets_client'; +import type { RuleSpecifier } from './types'; + +/** + * Retrieves the rule IDs (`rule_id`s) of available prebuilt rule assets matching those + * of the specified rules. This information can be used to determine whether + * the rule being imported is a custom rule or a prebuilt rule. + * + * @param rules - A list of {@link RuleSpecifier}s representing the rules being imported. + * @param ruleAssetsClient - the {@link IPrebuiltRuleAssetsClient} to use for fetching the available rule assets. + * + * @returns A list of the prebuilt rule asset IDs that are available. + * + */ +export const fetchAvailableRuleAssetIds = async ({ + rules, + ruleAssetsClient, +}: { + rules: RuleSpecifier[]; + ruleAssetsClient: IPrebuiltRuleAssetsClient; +}): Promise => { + const incomingRuleIds = rules.map((rule) => rule.rule_id); + const availableRuleAssets = await ruleAssetsClient.fetchLatestAssetsByRuleId(incomingRuleIds); + + return availableRuleAssets.map((asset) => asset.rule_id); +}; + +/** + * Retrieves prebuilt rule assets for rules being imported. These + * assets can be compared to the incoming rules for the purposes of calculating + * appropriate `rule_source` values. + * + * @param rules - A list of {@link RuleSpecifier}s representing the rules being imported. + * + * @returns The prebuilt rule assets matching the specified prebuilt + * rules. Assets match the `rule_id` and `version` of the specified rules. + * Because of this, there may be less assets returned than specified rules. + */ +export const fetchMatchingAssets = async ({ + rules, + ruleAssetsClient, +}: { + rules: RuleSpecifier[]; + ruleAssetsClient: IPrebuiltRuleAssetsClient; +}): Promise => { + const incomingRuleVersions = rules.flatMap((rule) => { + if (rule.version == null) { + return []; + } + return { + rule_id: rule.rule_id, + version: rule.version, + }; + }); + + return ruleAssetsClient.fetchAssetsByVersion(incomingRuleVersions); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 4a96b127df8a7..948d4edffb9ed 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -20,7 +20,8 @@ import type { ConfigType } from '../../../../../../config'; import type { HapiReadableStream, SecuritySolutionPluginRouter } from '../../../../../../types'; import type { BulkError, ImportRuleResponse } from '../../../../routes/utils'; import { buildSiemResponse, isBulkError, isImportRegular } from '../../../../routes/utils'; -import { PrebuiltRulesImportHelper } from '../../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; +import { createRuleSourceImporter } from '../../../../prebuilt_rules/logic/rule_source_importer'; +import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; import { importRules } from '../../../logic/import/import_rules_with_source'; import { importRules as legacyImportRules } from '../../../logic/import/import_rules_utils'; @@ -142,10 +143,10 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C ? [] : rulesWithMigratedActions || migratedParsedObjectsWithoutDuplicateErrors; - const prebuiltRulesImportHelper = new PrebuiltRulesImportHelper({ + const ruleSourceImporter = createRuleSourceImporter({ config, context: ctx.securitySolution, - savedObjectsClient, + prebuiltRuleAssetsClient: createPrebuiltRuleAssetsClient(savedObjectsClient), }); const chunkParseObjects = chunk(CHUNK_PARSED_OBJECT_SIZE, parsedRules); @@ -158,7 +159,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], overwriteRules: request.query.overwrite, allowMissingConnectorSecrets: !!actionConnectors.length, - prebuiltRulesImportHelper, + ruleSourceImporter, detectionRulesClient, }); } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts index 0d7810d74fcbb..1ce48106d0ae1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts @@ -12,7 +12,7 @@ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; import { buildMlAuthz } from '../../../../machine_learning/__mocks__/authz'; import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; -import { prebuiltRulesImportHelperMock } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper.mock'; +import { ruleSourceImporterMock } from '../../../prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock'; import { createDetectionRulesClient } from './detection_rules_client'; import { importRuleWithSource } from './methods/import_rule_with_source'; import { createRuleImportErrorObject } from '../import/errors'; @@ -24,7 +24,7 @@ jest.mock('../import/check_rule_exception_references'); describe('detectionRulesClient.importRules', () => { let subject: ReturnType; let ruleToImport: ReturnType; - let prebuiltRulesImportHelper: ReturnType; + let mockRuleSourceImporter: ReturnType; beforeEach(() => { subject = createDetectionRulesClient({ @@ -38,16 +38,18 @@ describe('detectionRulesClient.importRules', () => { (importRuleWithSource as jest.Mock).mockResolvedValue(getRulesSchemaMock()); ruleToImport = getImportRulesSchemaMock(); - prebuiltRulesImportHelper = prebuiltRulesImportHelperMock.create(); - prebuiltRulesImportHelper.fetchAssetRuleIds.mockResolvedValue([]); - prebuiltRulesImportHelper.fetchMatchingAssets.mockResolvedValue([]); + mockRuleSourceImporter = ruleSourceImporterMock.create(); + mockRuleSourceImporter.calculateRuleSource.mockReturnValue({ + ruleSource: { type: 'internal' }, + immutable: false, + }); }); it('returns imported rules as RuleResponses if import was successful', async () => { const result = await subject.importRules({ allowMissingConnectorSecrets: false, overwriteRules: false, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, rules: [ruleToImport, ruleToImport], }); @@ -64,7 +66,7 @@ describe('detectionRulesClient.importRules', () => { const result = await subject.importRules({ allowMissingConnectorSecrets: false, overwriteRules: false, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, rules: [ruleToImport], }); @@ -78,7 +80,7 @@ describe('detectionRulesClient.importRules', () => { const result = await subject.importRules({ allowMissingConnectorSecrets: false, overwriteRules: false, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, rules: [ruleToImport], }); @@ -110,7 +112,7 @@ describe('detectionRulesClient.importRules', () => { const result = await subject.importRules({ allowMissingConnectorSecrets: false, overwriteRules: false, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, rules: [ruleToImport], }); @@ -136,7 +138,7 @@ describe('detectionRulesClient.importRules', () => { const result = await subject.importRules({ allowMissingConnectorSecrets: false, overwriteRules: false, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, rules: [ruleToImport], }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts index 517bebddc60ee..2aa57ed5b95d0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts @@ -15,9 +15,9 @@ import type { RuleToImport, RuleSource, } from '../../../../../../common/api/detection_engine'; +import type { IRuleSourceImporter } from '../../../prebuilt_rules/logic/rule_source_importer'; import type { RuleImportErrorObject } from '../import/errors'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; -import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; export interface IDetectionRulesClient { createCustomRule: (args: CreateCustomRuleArgs) => Promise; @@ -70,6 +70,6 @@ export interface ImportRuleArgs { export interface ImportRulesArgs { rules: RuleToImport[]; overwriteRules: boolean; - prebuiltRulesImportHelper: PrebuiltRulesImportHelper; + ruleSourceImporter: IRuleSourceImporter; allowMissingConnectorSecrets?: boolean; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts index ead7456d5b9b5..42ef6c621e93e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts @@ -10,14 +10,13 @@ import type { SavedObjectsClientContract } from '@kbn/core/server'; import type { RuleResponse, RuleToImport } from '../../../../../../../common/api/detection_engine'; import { ruleToImportHasVersion } from '../../../../../../../common/api/detection_engine/rule_management'; -import type { PrebuiltRulesImportHelper } from '../../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; +import type { IRuleSourceImporter } from '../../../../prebuilt_rules/logic/rule_source_importer'; import { type RuleImportErrorObject, createRuleImportErrorObject, isRuleImportError, } from '../../import/errors'; import { checkRuleExceptionReferences } from '../../import/check_rule_exception_references'; -import { calculateRuleSourceForImport } from '../../import/calculate_rule_source_for_import'; import { getReferencedExceptionLists } from '../../import/gather_referenced_exceptions'; import type { IDetectionRulesClient } from '../detection_rules_client_interface'; @@ -29,14 +28,14 @@ export const importRules = async ({ allowMissingConnectorSecrets, detectionRulesClient, overwriteRules, - prebuiltRulesImportHelper, + ruleSourceImporter, rules, savedObjectsClient, }: { allowMissingConnectorSecrets?: boolean; detectionRulesClient: IDetectionRulesClient; overwriteRules: boolean; - prebuiltRulesImportHelper: PrebuiltRulesImportHelper; + ruleSourceImporter: IRuleSourceImporter; rules: RuleToImport[]; savedObjectsClient: SavedObjectsClientContract; }): Promise> => { @@ -44,38 +43,33 @@ export const importRules = async ({ rules, savedObjectsClient, }); - const prebuiltRuleAssets = await prebuiltRulesImportHelper.fetchMatchingAssets({ - rules, - }); - - const installedRuleIds = await prebuiltRulesImportHelper.fetchAssetRuleIds({ - rules, - }); + await ruleSourceImporter.setup({ rules }); return Promise.all( rules.map(async (rule) => { - if (!ruleToImportHasVersion(rule)) { - return createRuleImportErrorObject({ - message: i18n.translate( - 'xpack.securitySolution.detectionEngine.rules.cannotImportRuleWithoutVersion', - { - defaultMessage: 'Rules must specify a "version" to be imported. [rule_id: {ruleId}]', - values: { ruleId: rule.rule_id }, - } - ), - ruleId: rule.rule_id, - }); - } - const errors: RuleImportErrorObject[] = []; - const { immutable, ruleSource } = calculateRuleSourceForImport({ - rule, - prebuiltRuleAssets, - installedRuleIds, - }); - try { + if (!ruleSourceImporter.isPrebuiltRule(rule)) { + rule.version = rule.version ?? 1; + } + + if (!ruleToImportHasVersion(rule)) { + return createRuleImportErrorObject({ + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.cannotImportPrebuiltRuleWithoutVersion', + { + defaultMessage: + 'Prebuilt rules must specify a "version" to be imported. [rule_id: {ruleId}]', + values: { ruleId: rule.rule_id }, + } + ), + ruleId: rule.rule_id, + }); + } + + const { immutable, ruleSource } = ruleSourceImporter.calculateRuleSource(rule); + const [exceptionErrors, exceptions] = checkRuleExceptionReferences({ rule, existingLists, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts index 19a77db0e0c86..d0e1019819ac2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.ts @@ -14,7 +14,7 @@ import type { } from '@kbn/securitysolution-io-ts-list-types'; import { createRulesAndExceptionsStreamFromNdJson } from './create_rules_stream_from_ndjson'; -import type { RuleFromImportStream } from './types'; +import type { RuleFromImportStream } from './utils'; export interface RuleImportStreamResult { rules: RuleFromImportStream[]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 435ae5e722f79..0b1fb6f20036f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -13,7 +13,7 @@ import { createBulkErrorObject } from '../../../routes/utils'; import { checkRuleExceptionReferences } from './check_rule_exception_references'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { getReferencedExceptionLists } from './gather_referenced_exceptions'; -import type { RuleFromImportStream } from './types'; +import type { RuleFromImportStream } from './utils'; /** * Takes rules to be imported and either creates or updates rules diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts index 88742354d9abc..a3b66f7d6bb45 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts @@ -7,7 +7,7 @@ import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; -import { prebuiltRulesImportHelperMock } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper.mock'; +import { ruleSourceImporterMock } from '../../../prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock'; import { importRules } from './import_rules_with_source'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; @@ -18,7 +18,7 @@ describe('importRules', () => { let ruleToImport: ReturnType; let detectionRulesClient: jest.Mocked; - let prebuiltRulesImportHelper: ReturnType; + let mockRuleSourceImporter: ReturnType; beforeEach(() => { jest.clearAllMocks(); @@ -26,9 +26,7 @@ describe('importRules', () => { detectionRulesClient = detectionRulesClientMock.create(); detectionRulesClient.importRules.mockResolvedValue([]); ruleToImport = getImportRulesSchemaMock(); - prebuiltRulesImportHelper = prebuiltRulesImportHelperMock.create(); - prebuiltRulesImportHelper.fetchMatchingAssets.mockResolvedValue([]); - prebuiltRulesImportHelper.fetchAssetRuleIds.mockResolvedValue([]); + mockRuleSourceImporter = ruleSourceImporterMock.create(); }); it('returns an empty rules response if no rules to import', async () => { @@ -37,7 +35,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, }); expect(result).toEqual([]); @@ -51,7 +49,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, }); expect(result).toEqual([ @@ -89,7 +87,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, }); expect(result).toEqual([ @@ -129,7 +127,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, }); expect(result).toEqual([ @@ -171,7 +169,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, }); expect(result).toEqual([ @@ -206,7 +204,7 @@ describe('importRules', () => { rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, - prebuiltRulesImportHelper, + ruleSourceImporter: mockRuleSourceImporter, }); expect(result).toEqual([ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts index a8d7e6516d6cb..4dbadee0f1088 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts @@ -8,12 +8,10 @@ import { partition } from 'lodash'; import { type ImportRuleResponse, createBulkErrorObject } from '../../../routes/utils'; -import type { PrebuiltRulesImportHelper } from '../../../prebuilt_rules/logic/prebuilt_rules_import_helper'; +import type { IRuleSourceImporter } from '../../../prebuilt_rules/logic/rule_source_importer'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { isRuleConflictError, isRuleImportError } from './errors'; -import type { RuleFromImportStream } from './types'; - -const ruleIsError = (rule: RuleFromImportStream): rule is Error => rule instanceof Error; +import { isRuleToImport, type RuleFromImportStream } from './utils'; /** * Takes a stream of rules to be imported and either creates or updates rules @@ -34,14 +32,14 @@ export const importRules = async ({ rulesResponseAcc, overwriteRules, detectionRulesClient, - prebuiltRulesImportHelper, + ruleSourceImporter, allowMissingConnectorSecrets, }: { ruleChunks: RuleFromImportStream[][]; rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; - prebuiltRulesImportHelper: PrebuiltRulesImportHelper; + ruleSourceImporter: IRuleSourceImporter; allowMissingConnectorSecrets?: boolean; }) => { let response: ImportRuleResponse[] = [...rulesResponseAcc]; @@ -52,16 +50,14 @@ export const importRules = async ({ return response; } - await prebuiltRulesImportHelper.setup(); - while (ruleChunks.length) { const ruleChunk = ruleChunks.shift() ?? []; - const [errors, rules] = partition(ruleChunk, ruleIsError); + const [rules, errors] = partition(ruleChunk, isRuleToImport); const importedRulesResponse = await detectionRulesClient.importRules({ allowMissingConnectorSecrets, overwriteRules, - prebuiltRulesImportHelper, + ruleSourceImporter, rules, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/utils.ts similarity index 78% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/types.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/utils.ts index b9bae0b23223b..b25efe7d9390d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/utils.ts @@ -8,3 +8,6 @@ import type { RuleToImport } from '../../../../../../common/api/detection_engine/rule_management'; export type RuleFromImportStream = RuleToImport | Error; + +export const isRuleToImport = (rule: RuleFromImportStream): rule is RuleToImport => + !(rule instanceof Error); From 79767e2f3932ed25bd8f1bb146fd2e7748cde433 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 19 Sep 2024 22:59:47 -0500 Subject: [PATCH 084/120] Fix some unit tests related to the route/client error refactor See 1053da5408a9b8bed3b4433c051568b16cab7542 for details. --- .../detection_rules_client.import_rule.test.ts | 4 ++-- .../methods/import_rule_with_source.test.ts | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts index 7b64d6daa76de..f5ade2722747f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts @@ -237,10 +237,10 @@ describe('DetectionRulesClient.importRule', () => { }) ).rejects.toMatchObject({ error: { - status_code: 409, + ruleId: ruleToImport.rule_id, + type: 'conflict', message: `rule_id: "${ruleToImport.rule_id}" already exists`, }, - rule_id: ruleToImport.rule_id, }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts index b27133ea74b0d..615cd007d2f46 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts @@ -73,8 +73,9 @@ describe('importRuleWithSource', () => { }) ).rejects.toMatchObject({ error: { + type: 'conflict', + ruleId: 'rule-1', message: 'rule_id: "rule-1" already exists', - status_code: 409, }, }); }); From 4396c2b7c62eba2629367441b11f39e6c4887867 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 19 Sep 2024 23:03:35 -0500 Subject: [PATCH 085/120] Remove needless _with_source suffix on filename The existing `importRules` function (which is now significantly different) was already located in the `import_rules_utils` file, so we don't need this distinction. --- .../rule_management/api/rules/import_rules/route.ts | 2 +- .../{import_rules_with_source.test.ts => import_rules.test.ts} | 2 +- .../import/{import_rules_with_source.ts => import_rules.ts} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/{import_rules_with_source.test.ts => import_rules.test.ts} (99%) rename x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/{import_rules_with_source.ts => import_rules.ts} (100%) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 948d4edffb9ed..5701663eb5925 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -23,7 +23,7 @@ import { buildSiemResponse, isBulkError, isImportRegular } from '../../../../rou import { createRuleSourceImporter } from '../../../../prebuilt_rules/logic/rule_source_importer'; import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; -import { importRules } from '../../../logic/import/import_rules_with_source'; +import { importRules } from '../../../logic/import/import_rules'; import { importRules as legacyImportRules } from '../../../logic/import/import_rules_utils'; import { createPromiseFromRuleImportStream } from '../../../logic/import/create_promise_from_rule_import_stream'; import { importRuleExceptions } from '../../../logic/import/import_rule_exceptions'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts similarity index 99% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts index a3b66f7d6bb45..4836d263256ce 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts @@ -9,7 +9,7 @@ import { getImportRulesSchemaMock } from '../../../../../../common/api/detection import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; import { ruleSourceImporterMock } from '../../../prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock'; -import { importRules } from './import_rules_with_source'; +import { importRules } from './import_rules'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { detectionRulesClientMock } from '../detection_rules_client/__mocks__/detection_rules_client'; import { createRuleImportErrorObject } from './errors'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_with_source.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts From 3102745f4a4a83fbf2e41497c58bd369d3e8e151 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 20 Sep 2024 15:53:48 -0500 Subject: [PATCH 086/120] Fix handling of versionless rules in RuleSourceImporter I had an edge case where the importer would complain if you called `setup()` with a versionless rule, and then called into the importer again with a versioned copy of that rule. --- .../rule_source_importer.test.ts | 28 ++++++++++++++++--- .../rule_source_importer.ts | 4 ++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts index 3ad25a526c464..07df3f6fa10e8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts @@ -21,6 +21,7 @@ describe('ruleSourceImporter', () => { let subject: ReturnType; beforeEach(() => { + jest.clearAllMocks(); config = createMockConfig(); config = configMock.withExperimentalFeature(config, 'prebuiltRulesCustomizationEnabled'); context = requestContextMock.create().securitySolution; @@ -108,15 +109,16 @@ describe('ruleSourceImporter', () => { beforeEach(() => { rule = { rule_id: 'validated-rule', version: 1 } as ValidatedRuleToImport; - }); - - it('invokes calculateRuleSourceForImport with the correct arguments', async () => { ruleAssetsClientMock.fetchAssetsByVersion.mockResolvedValue([ getPrebuiltRuleMock({ rule_id: 'rule-1' }), ]); ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([ + getPrebuiltRuleMock({ rule_id: 'rule-1' }), getPrebuiltRuleMock({ rule_id: 'rule-2' }), ]); + }); + + it('invokes calculateRuleSourceForImport with the correct arguments', async () => { const calculatorSpy = jest .spyOn(calculateRuleSourceModule, 'calculateRuleSourceForImport') .mockReturnValue({ ruleSource: { type: 'internal' }, immutable: false }); @@ -128,7 +130,7 @@ describe('ruleSourceImporter', () => { expect(calculatorSpy).toHaveBeenCalledWith({ rule, prebuiltRuleAssets: [expect.objectContaining({ rule_id: 'rule-1' })], - installedRuleIds: ['rule-2'], + installedRuleIds: ['rule-1', 'rule-2'], }); }); @@ -146,5 +148,23 @@ describe('ruleSourceImporter', () => { `"Rule validated-rule was not registered during setup."` ); }); + + describe('for rules set up without a version', () => { + it('invokes the calculator with the correct arguments', async () => { + const calculatorSpy = jest + .spyOn(calculateRuleSourceModule, 'calculateRuleSourceForImport') + .mockReturnValue({ ruleSource: { type: 'internal' }, immutable: false }); + + await subject.setup({ rules: [{ ...rule, version: undefined }] }); + await subject.calculateRuleSource(rule); + + expect(calculatorSpy).toHaveBeenCalledTimes(1); + expect(calculatorSpy).toHaveBeenCalledWith({ + rule, + prebuiltRuleAssets: [expect.objectContaining({ rule_id: 'rule-1' })], + installedRuleIds: ['rule-1', 'rule-2'], + }); + }); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts index fcdf27b472d40..dda728d673997 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts @@ -145,7 +145,9 @@ export class RuleSourceImporter implements IRuleSourceImporter { private validateRuleInput(rule: RuleToImport) { if ( !this.knownRules.some( - (knownRule) => knownRule.rule_id === rule.rule_id && knownRule.version === rule.version + (knownRule) => + knownRule.rule_id === rule.rule_id && + (knownRule.version === rule.version || knownRule.version == null) ) ) { throw new Error(`Rule ${rule.rule_id} was not registered during setup.`); From 5c275ac6842a7612eb2fbc679a0658fa5b50232f Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 20 Sep 2024 15:55:14 -0500 Subject: [PATCH 087/120] Add integration tests around new import functionality This covers most of the permutations of import with rule customization, including importing a rule identical to an asset. --- .../import_rules.ts | 167 ++++++++++++++++-- 1 file changed, 151 insertions(+), 16 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts index 16239f12bebc5..934ee6460a5e2 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/prebuilt_rule_customization/trial_license_complete_tier/import_rules.ts @@ -7,44 +7,179 @@ import expect from 'expect'; -import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants'; -import { combineToNdJson, getCustomQueryRuleParams } from '../../../../utils'; +import { + SAMPLE_PREBUILT_RULES_WITH_HISTORICAL_VERSIONS, + combineArrayToNdJson, + createHistoricalPrebuiltRuleAssetSavedObjects, + deleteAllPrebuiltRuleAssets, + fetchRule, + getCustomQueryRuleParams, + getInstalledRules, +} from '../../../../utils'; import { deleteAllRules } from '../../../../../../../common/utils/security_solution'; import { FtrProviderContext } from '../../../../../../ftr_provider_context'; export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); + const es = getService('es'); const log = getService('log'); + const securitySolutionApi = getService('securitySolutionApi'); + + const importRules = async (rules: unknown[]) => { + const buffer = Buffer.from(combineArrayToNdJson(rules)); + + return securitySolutionApi + .importRules({ query: {} }) + .attach('file', buffer, 'rules.ndjson') + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(200); + }; + + const prebuiltRules = SAMPLE_PREBUILT_RULES_WITH_HISTORICAL_VERSIONS.map( + (prebuiltRule) => prebuiltRule['security-rule'] + ); + const prebuiltRuleIds = [...new Set(prebuiltRules.map((rule) => rule.rule_id))]; describe('@ess @serverless @skipInServerlessMKI import_rules', () => { + before(async () => { + await deleteAllPrebuiltRuleAssets(es, log); + await createHistoricalPrebuiltRuleAssetSavedObjects( + es, + SAMPLE_PREBUILT_RULES_WITH_HISTORICAL_VERSIONS + ); + }); + + after(async () => { + await deleteAllPrebuiltRuleAssets(es, log); + await deleteAllRules(supertest, log); + }); + beforeEach(async () => { await deleteAllRules(supertest, log); }); - describe('calculation of the rule_source fields', () => { - // TODO if we do this, how do distinguish a custom rule from a prebuilt one? - it('calculates a version of 1 for custom rules'); + describe('calculation of rule customization fields', () => { + it('defaults a versionless custom rule to "version: 1"', async () => { + const rule = getCustomQueryRuleParams({ rule_id: 'custom-rule', version: undefined }); + const { body } = await importRules([rule]); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + + const importedRule = await fetchRule(supertest, { ruleId: rule.rule_id! }); + expect(importedRule).toMatchObject({ + rule_id: rule.rule_id, + version: 1, + rule_source: { type: 'internal' }, + immutable: false, + }); + }); + + it('preserves a custom rule with a specified version', async () => { + const rule = getCustomQueryRuleParams({ rule_id: 'custom-rule', version: 23 }); + const { body } = await importRules([rule]); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); - it('rejects a prebuilt rule with an unspecified version', async () => { - const rule = getCustomQueryRuleParams(); - delete rule.version; - const ndjson = combineToNdJson(rule); + const importedRule = await fetchRule(supertest, { ruleId: rule.rule_id! }); + expect(importedRule).toMatchObject({ + rule_id: rule.rule_id, + version: 23, + rule_source: { type: 'internal' }, + immutable: false, + }); + }); - const { body } = await supertest - .post(`${DETECTION_ENGINE_RULES_URL}/_import`) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '2023-10-31') - .attach('file', Buffer.from(ndjson), 'rules.ndjson') - .expect(200); + it('rejects a versionless prebuilt rule', async () => { + const rule = getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[0], version: undefined }); + const { body } = await importRules([rule]); expect(body.errors).toHaveLength(1); expect(body.errors[0]).toMatchObject({ error: { - message: 'Rules must specify a "version" to be imported. [rule_id: rule-1]', + message: `Prebuilt rules must specify a "version" to be imported. [rule_id: ${prebuiltRuleIds[0]}]`, status_code: 400, }, }); }); + + it('respects the version of a prebuilt rule', async () => { + const rule = getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[1], version: 9999 }); + const { body } = await importRules([rule]); + + expect(body).toMatchObject({ + rules_count: 1, + success: true, + success_count: 1, + errors: [], + }); + + const importedRule = await fetchRule(supertest, { ruleId: rule.rule_id! }); + expect(importedRule).toMatchObject({ + rule_id: rule.rule_id, + version: 9999, + rule_source: { type: 'external', is_customized: true }, + immutable: true, + }); + }); + + it('imports a combination of prebuilt and custom rules', async () => { + const rules = [ + getCustomQueryRuleParams({ rule_id: 'custom-rule', version: 23 }), + getCustomQueryRuleParams({ rule_id: prebuiltRuleIds[0], version: 1234 }), + getCustomQueryRuleParams({ rule_id: 'custom-rule-2', version: undefined }), + prebuiltRules[3], + ]; + const { body } = await importRules(rules); + + expect(body).toMatchObject({ + rules_count: 4, + success: true, + success_count: 4, + errors: [], + }); + + const { data: importedRules } = await getInstalledRules(supertest); + + expect(importedRules).toHaveLength(4); + expect(importedRules).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + rule_id: 'custom-rule', + version: 23, + rule_source: { type: 'internal' }, + immutable: false, + }), + expect.objectContaining({ + rule_id: prebuiltRuleIds[0], + version: 1234, + rule_source: { type: 'external', is_customized: true }, + immutable: true, + }), + expect.objectContaining({ + rule_id: 'custom-rule-2', + version: 1, + rule_source: { type: 'internal' }, + immutable: false, + }), + expect.objectContaining({ + rule_id: prebuiltRules[3].rule_id, + version: prebuiltRules[3].version, + rule_source: { type: 'external', is_customized: false }, + immutable: true, + }), + ]) + ); + }); }); }); }; From 60cfedac306e621b5a8a436a726acee17cdfe3a6 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 20 Sep 2024 16:10:02 -0500 Subject: [PATCH 088/120] Move calculateRuleSourceFromAsset to more appropriate location It's not part of the rules client, so we're putting it into the import logic where it belongs. --- .../logic/import/calculate_rule_source_for_import.ts | 2 +- .../calculate_rule_source_from_asset.test.ts | 4 ++-- .../calculate_rule_source_from_asset.ts | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) rename x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/{detection_rules_client/mergers/rule_source => import}/calculate_rule_source_from_asset.test.ts (92%) rename x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/{detection_rules_client/mergers/rule_source => import}/calculate_rule_source_from_asset.ts (79%) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index d2146bf047d8c..38669e67a9665 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -10,7 +10,7 @@ import type { ValidatedRuleToImport, } from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; -import { calculateRuleSourceFromAsset } from '../detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset'; +import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset'; /** * Calculates the rule_source field for a rule being imported diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts similarity index 92% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts index 570abab48c4c6..7d5b539f357d8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts @@ -6,8 +6,8 @@ */ import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset'; -import { getRulesSchemaMock } from '../../../../../../../../common/api/detection_engine/model/rule_schema/mocks'; -import { getPrebuiltRuleMock } from '../../../../../prebuilt_rules/mocks'; +import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; +import { getPrebuiltRuleMock } from '../../../prebuilt_rules/mocks'; describe('calculateRuleSourceFromAsset', () => { it('calculates as internal if no asset is found', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts similarity index 79% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts index 7766b286f8e2e..0b45139e5aa08 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts @@ -5,10 +5,10 @@ * 2.0. */ -import type { RuleSource } from '../../../../../../../../common/api/detection_engine'; -import type { DiffableRuleInput } from '../../../../../../../../common/detection_engine/prebuilt_rules/diff/types'; -import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; -import { calculateIsCustomized } from './calculate_is_customized'; +import type { RuleSource } from '../../../../../../common/api/detection_engine'; +import type { DiffableRuleInput } from '../../../../../../common/detection_engine/prebuilt_rules/diff/types'; +import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; +import { calculateIsCustomized } from '../detection_rules_client/mergers/rule_source/calculate_is_customized'; /** * Calculates rule_source for a rule based on two pieces of information: From 46e641d04be8bde3ef1c7f567f719596fea4fd73 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 20 Sep 2024 16:32:50 -0500 Subject: [PATCH 089/120] Remove new method of PrebuiltRuleAssetsClient in favor of extending existing method Since for our purposes we only need the `rule_ids` of the assets specified, we can extend the existing `fetchLatestVersions` method to optionally apply a filter of `rule_id`s to the query, and do away with the specialized method I had previously introduced. --- .../__mocks__/prebuilt_rule_assets_client.ts | 1 - .../prebuilt_rule_assets_client.ts | 56 +------------------ .../rule_source_importer.test.ts | 10 ++-- .../logic/rule_source_importer/utils.ts | 4 +- 4 files changed, 10 insertions(+), 61 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client.ts index b9233a121ccf6..0776fefb98656 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client.ts @@ -8,7 +8,6 @@ export const createPrebuiltRuleAssetsClient = () => { return { fetchLatestAssets: jest.fn(), - fetchLatestAssetsByRuleId: jest.fn(), fetchLatestVersions: jest.fn(), fetchAssetsByVersion: jest.fn(), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts index fd0fdac62cdbf..cc76200ee39bd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts @@ -20,9 +20,7 @@ const MAX_PREBUILT_RULES_COUNT = 10_000; export interface IPrebuiltRuleAssetsClient { fetchLatestAssets: () => Promise; - fetchLatestAssetsByRuleId(ruleIds: string[]): Promise; - - fetchLatestVersions(): Promise; + fetchLatestVersions(ruleIds?: string[]): Promise; fetchAssetsByVersion(versions: RuleVersionSpecifier[]): Promise; } @@ -74,13 +72,8 @@ export const createPrebuiltRuleAssetsClient = ( }); }, - fetchLatestAssetsByRuleId: (ruleIds: string[]) => { - return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchLatestAssetsByRuleId', async () => { - if (ruleIds.length === 0) { - // NOTE: without early return it would build incorrect filter and fetch all existing saved objects - return []; - } - + fetchLatestVersions: (ruleIds: string[] = []): Promise => { + return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchLatestVersions', async () => { const filter = ruleIds .map((ruleId) => `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id: ${ruleId}`) .join(' OR '); @@ -95,49 +88,6 @@ export const createPrebuiltRuleAssetsClient = ( >({ type: PREBUILT_RULE_ASSETS_SO_TYPE, filter, - aggs: { - rules: { - terms: { - field: `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id`, - size: MAX_PREBUILT_RULES_COUNT, - }, - aggs: { - latest_version: { - top_hits: { - size: 1, - sort: { - [`${PREBUILT_RULE_ASSETS_SO_TYPE}.version`]: 'desc', - }, - }, - }, - }, - }, - }, - }); - - const buckets = findResult.aggregations?.rules?.buckets ?? []; - invariant(Array.isArray(buckets), 'Expected buckets to be an array'); - - const ruleAssets = buckets.map((bucket) => { - const hit = bucket.latest_version.hits.hits[0]; - return hit._source[PREBUILT_RULE_ASSETS_SO_TYPE]; - }); - - return validatePrebuiltRuleAssets(ruleAssets); - }); - }, - - fetchLatestVersions: (): Promise => { - return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchLatestVersions', async () => { - const findResult = await savedObjectsClient.find< - PrebuiltRuleAsset, - { - rules: AggregationsMultiBucketAggregateBase<{ - latest_version: AggregationsTopHitsAggregate; - }>; - } - >({ - type: PREBUILT_RULE_ASSETS_SO_TYPE, aggs: { rules: { terms: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts index 07df3f6fa10e8..7b4c6bffaebea 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts @@ -75,21 +75,21 @@ describe('ruleSourceImporter', () => { beforeEach(() => {}); it("returns false if the rule's rule_id doesn't match an available rule asset", async () => { - ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([]); + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([]); await subject.setup({ rules: [ruleToImport] }); expect(subject.isPrebuiltRule(ruleToImport)).toBe(false); }); it("returns true if the rule's rule_id matches an available rule asset", async () => { - ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([ruleToImport]); + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ruleToImport]); await subject.setup({ rules: [ruleToImport] }); expect(subject.isPrebuiltRule(ruleToImport)).toBe(true); }); it('throws an error if the rule is not known to the calculator', async () => { - ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([ruleToImport]); + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ruleToImport]); await subject.setup({ rules: [ruleToImport] }); expect(() => @@ -112,7 +112,7 @@ describe('ruleSourceImporter', () => { ruleAssetsClientMock.fetchAssetsByVersion.mockResolvedValue([ getPrebuiltRuleMock({ rule_id: 'rule-1' }), ]); - ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([ + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ getPrebuiltRuleMock({ rule_id: 'rule-1' }), getPrebuiltRuleMock({ rule_id: 'rule-2' }), ]); @@ -135,7 +135,7 @@ describe('ruleSourceImporter', () => { }); it('throws an error if the rule is not known to the calculator', async () => { - ruleAssetsClientMock.fetchLatestAssetsByRuleId.mockResolvedValue([ruleToImport]); + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ruleToImport]); await subject.setup({ rules: [ruleToImport] }); expect(() => subject.calculateRuleSource(rule)).toThrowErrorMatchingInlineSnapshot( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts index 41d3a71825f2f..da59a2d1e649c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts @@ -28,9 +28,9 @@ export const fetchAvailableRuleAssetIds = async ({ ruleAssetsClient: IPrebuiltRuleAssetsClient; }): Promise => { const incomingRuleIds = rules.map((rule) => rule.rule_id); - const availableRuleAssets = await ruleAssetsClient.fetchLatestAssetsByRuleId(incomingRuleIds); + const availableRuleAssetSpecifiers = await ruleAssetsClient.fetchLatestVersions(incomingRuleIds); - return availableRuleAssets.map((asset) => asset.rule_id); + return availableRuleAssetSpecifiers.map((specifier) => specifier.rule_id); }; /** From c798d0e4431d36ddcb99d4eef7a1946e3a53eb8f Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 20 Sep 2024 17:23:13 -0500 Subject: [PATCH 090/120] Aligning importRule implementations, part 1 The first step here is to combine my unit tests for `detectionRulesClient.importRule` and `importRuleWithSource`). After that, we can swap out implementations back to `importRule`, modify things to use an overriding mechanism, and we should be set. --- ...detection_rules_client.import_rule.test.ts | 175 +++++++++- .../methods/import_rule_with_source.test.ts | 326 ------------------ 2 files changed, 174 insertions(+), 327 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts index f5ade2722747f..cc3da7fb21f96 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts @@ -17,7 +17,10 @@ import { getQueryRuleParams } from '../../../rule_schema/mocks'; import { createDetectionRulesClient } from './detection_rules_client'; import type { IDetectionRulesClient } from './detection_rules_client_interface'; import { getRuleByRuleId } from './methods/get_rule_by_rule_id'; -import { getValidatedRuleToImportWithSourceMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; +import { + getValidatedRuleToImportMock, + getValidatedRuleToImportWithSourceMock, +} from '../../../../../../common/api/detection_engine/rule_management/mocks'; jest.mock('../../../../machine_learning/authz'); jest.mock('../../../../machine_learning/validation'); @@ -243,5 +246,175 @@ describe('DetectionRulesClient.importRule', () => { }, }); }); + + it("always uses the existing rule's 'id' value", async () => { + const rule = { + ...getValidatedRuleToImportMock(), + immutable: true, + id: 'some-id', + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).not.toHaveBeenCalled(); + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + id: existingRule.id, + }) + ); + }); + + it("uses the existing rule's 'version' value if not unspecified", async () => { + const rule = { + ...getValidatedRuleToImportMock(), + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).not.toHaveBeenCalled(); + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + version: existingRule.version, + }), + }), + }) + ); + }); + + it("uses the specified 'version' value", async () => { + const rule = { + ...getValidatedRuleToImportMock(), + immutable: true, + version: 42, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).not.toHaveBeenCalled(); + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + version: rule.version, + }), + }), + }) + ); + }); + }); + + describe('when importing a new rule', () => { + beforeEach(() => { + (getRuleByRuleId as jest.Mock).mockReset().mockResolvedValueOnce(null); + }); + + it('preserves the passed "rule_source" and "immutable" values', async () => { + const rule = { + ...getValidatedRuleToImportMock(), + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + immutable: true, + ruleSource: { + isCustomized: true, + type: 'external', + }, + }), + }), + }) + ); + }); + + it('preserves the passed "enabled" value', async () => { + const rule = { + ...getValidatedRuleToImportMock(), + enabled: true, + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + enabled: true, + }), + }) + ); + }); + + it('defaults defaultable values', async () => { + const rule = { + ...getValidatedRuleToImportMock(), + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + actions: [], + }), + }) + ); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts deleted file mode 100644 index 615cd007d2f46..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.test.ts +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { actionsClientMock } from '@kbn/actions-plugin/server/actions_client/actions_client.mock'; -import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; - -import { getValidatedRuleToImportMock } from '../../../../../../../common/api/detection_engine/rule_management/mocks'; -import { importRuleWithSource } from './import_rule_with_source'; -import { buildMlAuthz } from '../../../../../machine_learning/authz'; -import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; -import { - getEmptyFindResult, - getFindResultWithSingleHit, - getRuleMock, -} from '../../../../routes/__mocks__/request_responses'; -import { getQueryRuleParams } from '../../../../rule_schema/mocks'; - -jest.mock('../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'); -jest.mock('../../../../../machine_learning/authz'); - -describe('importRuleWithSource', () => { - let mockActionsClient: ReturnType; - let mockRulesClient: ReturnType; - let mockMlAuthz: ReturnType; - let mockPrebuiltRuleAssetsClient: ReturnType; - - let params: Omit[0], 'importRulePayload'>; - - beforeEach(() => { - mockActionsClient = actionsClientMock.create(); - mockRulesClient = rulesClientMock.create(); - mockRulesClient.create.mockResolvedValue(getRuleMock(getQueryRuleParams())); - mockRulesClient.update.mockResolvedValue(getRuleMock(getQueryRuleParams())); - mockMlAuthz = (buildMlAuthz as jest.Mock)(); - mockPrebuiltRuleAssetsClient = (createPrebuiltRuleAssetsClient as jest.Mock)(); - params = { - actionsClient: mockActionsClient, - rulesClient: mockRulesClient, - prebuiltRuleAssetClient: mockPrebuiltRuleAssetsClient, - mlAuthz: mockMlAuthz, - }; - }); - - describe('when importing an existing rule', () => { - let existingRule: ReturnType['data'][0]; - - beforeEach(() => { - mockRulesClient.find.mockResolvedValue(getFindResultWithSingleHit()); - [existingRule] = getFindResultWithSingleHit().data; - }); - - it('throws an error if overwriting is disabled', async () => { - const rule = { - ...getValidatedRuleToImportMock(), - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, - }; - - await expect( - importRuleWithSource({ - ...params, - importRulePayload: { - ruleToImport: rule, - overwriteRules: false, - }, - }) - ).rejects.toMatchObject({ - error: { - type: 'conflict', - ruleId: 'rule-1', - message: 'rule_id: "rule-1" already exists', - }, - }); - }); - - it('preserves the passed "rule_source" and "immutable" values', async () => { - const rule = { - ...getValidatedRuleToImportMock(), - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, - }; - - await importRuleWithSource({ - ...params, - importRulePayload: { - ruleToImport: rule, - overwriteRules: true, - }, - }); - - expect(mockRulesClient.update).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - params: expect.objectContaining({ - immutable: true, - ruleSource: { - isCustomized: true, - type: 'external', - }, - }), - }), - }) - ); - }); - - it('TODO REVIEW does not preserve the passed "enabled" value', async () => { - const disabledRule = { - ...getValidatedRuleToImportMock(), - enabled: false, - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, - }; - - await importRuleWithSource({ - ...params, - importRulePayload: { - ruleToImport: disabledRule, - overwriteRules: true, - }, - }); - - expect(mockRulesClient.update).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.not.objectContaining({ - enabled: expect.anything(), - }), - }) - ); - }); - - it('TODO REVIEW preserves the existing rule\'s "id" value', async () => { - const rule = { - ...getValidatedRuleToImportMock(), - immutable: true, - id: 'some-id', - rule_source: { - type: 'external' as const, - is_customized: true, - }, - }; - - await importRuleWithSource({ - ...params, - importRulePayload: { - ruleToImport: rule, - overwriteRules: true, - }, - }); - - expect(mockRulesClient.update).toHaveBeenCalledWith( - expect.objectContaining({ - id: existingRule.id, - data: expect.objectContaining({ - params: expect.objectContaining({ - version: existingRule.params.version, - }), - }), - }) - ); - }); - - it('TODO REVIEW preserves the existing rule\'s "version" value if left unspecified', async () => { - const rule = { - ...getValidatedRuleToImportMock(), - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, - }; - - await importRuleWithSource({ - ...params, - importRulePayload: { - ruleToImport: rule, - overwriteRules: true, - }, - }); - - expect(mockRulesClient.update).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - params: expect.objectContaining({ - version: existingRule.params.version, - }), - }), - }) - ); - }); - - it('TODO REVIEW preserves the specified "version" value', async () => { - const rule = { - ...getValidatedRuleToImportMock(), - immutable: true, - version: 42, - rule_source: { - type: 'external' as const, - is_customized: true, - }, - }; - - await importRuleWithSource({ - ...params, - importRulePayload: { - ruleToImport: rule, - overwriteRules: true, - }, - }); - - expect(mockRulesClient.update).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - params: expect.objectContaining({ - version: 42, - }), - }), - }) - ); - }); - }); - - describe('when importing a new rule', () => { - beforeEach(() => { - mockRulesClient.find.mockResolvedValueOnce(getEmptyFindResult()); - }); - - it('preserves the passed "rule_source" and "immutable" values', async () => { - const rule = { - ...getValidatedRuleToImportMock(), - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, - }; - - await importRuleWithSource({ - ...params, - importRulePayload: { - ruleToImport: rule, - }, - }); - - expect(mockRulesClient.create).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - params: expect.objectContaining({ - immutable: true, - ruleSource: { - isCustomized: true, - type: 'external', - }, - }), - }), - }) - ); - }); - - it('preserves the passed "enabled" value', async () => { - const rule = { - ...getValidatedRuleToImportMock(), - enabled: true, - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, - }; - - await importRuleWithSource({ - ...params, - importRulePayload: { - ruleToImport: rule, - overwriteRules: true, - }, - }); - - expect(mockRulesClient.create).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - enabled: true, - }), - }) - ); - }); - - it('defaults defaultable values', async () => { - const rule = { - ...getValidatedRuleToImportMock(), - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, - }; - - await importRuleWithSource({ - ...params, - importRulePayload: { - ruleToImport: rule, - }, - }); - - expect(mockRulesClient.create).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - actions: [], - }), - }) - ); - }); - }); -}); From cfb709c1cb9268ec7bf567032655b5f6f86e8c09 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 20 Sep 2024 18:08:07 -0500 Subject: [PATCH 091/120] Remove legacyImportRule method in favor of extensible importRule method Both importRules functions now ultimately call `detectionRulesClient.importRule`, which is a wrapper for the extended `importRule` that allows a new `overrideFields` param. This commit fixes those two entrypoints (including handling of the new `RuleImportErrorObject`s thrown by the client), as well as: 1. Updating tests to reflect importing with `overrideFields` instead of the rule argument containing those fields, 2. Removing the unnecessary ValidatedRuleToImportWithSource type/mock, since the only place that now knows/cares about that is within importRules --- .../import_rules/rule_to_import.mock.ts | 16 +- .../import_rules/rule_to_import.ts | 16 -- .../api/rules/import_rules/route.test.ts | 8 +- .../__mocks__/detection_rules_client.ts | 1 - ...detection_rules_client.import_rule.test.ts | 82 +++--- ...etection_rules_client.import_rules.test.ts | 12 +- ...on_rules_client.legacy_import_rule.test.ts | 247 ------------------ .../detection_rules_client.ts | 16 +- .../detection_rules_client_interface.ts | 11 +- .../methods/import_rule.ts | 21 +- .../methods/import_rule_with_source.ts | 100 ------- .../methods/import_rules.ts | 3 +- .../logic/import/import_rules_utils.test.ts | 10 +- .../logic/import/import_rules_utils.ts | 14 +- 14 files changed, 90 insertions(+), 467 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.legacy_import_rule.test.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts index 94824a0aac2cd..2d98cace04b2b 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts @@ -5,11 +5,7 @@ * 2.0. */ -import type { - RuleToImport, - ValidatedRuleToImport, - ValidatedRuleToImportWithSource, -} from './rule_to_import'; +import type { RuleToImport, ValidatedRuleToImport } from './rule_to_import'; export const getImportRulesSchemaMock = (rewrites?: Partial): RuleToImport => ({ @@ -32,16 +28,6 @@ export const getValidatedRuleToImportMock = ( ...getImportRulesSchemaMock(overrides), }); -export const getValidatedRuleToImportWithSourceMock = ( - overrides?: Partial -): ValidatedRuleToImportWithSource => ({ - rule_source: { - type: 'internal', - }, - immutable: false, - ...getValidatedRuleToImportMock(overrides), -}); - export const getImportRulesWithIdSchemaMock = (ruleId = 'rule-1'): RuleToImport => ({ id: '6afb8ce1-ea94-4790-8653-fd0b021d2113', description: 'some description', diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index e97c9abf47259..0d69848b50175 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -13,7 +13,6 @@ import { RuleSignatureId, TypeSpecificCreateProps, RuleVersion, - RuleSource, } from '../../model/rule_schema'; /** @@ -58,18 +57,3 @@ export const ValidatedRuleToImport = RuleToImport.and( version: RuleVersion, }) ); - -/** - * This type represents new rules being imported once the prebuilt rule - * customization work is complete. In addition to having been validated to have - * `version`, they've also had their `rule_source` and `immutable` fields - * calculated, and are ready to be persisted. - */ -export type ValidatedRuleToImportWithSource = z.infer; -export type ValidatedRuleToImportWithSourceInput = z.input; -export const ValidatedRuleToImportWithSource = ValidatedRuleToImport.and( - z.object({ - rule_source: RuleSource, - immutable: z.boolean(), - }) -); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts index 5d4dbec256172..89fee3201e009 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts @@ -56,7 +56,7 @@ describe('Import rules route', () => { clients.rulesClient.find.mockResolvedValue(getEmptyFindResult()); // no extant rules clients.rulesClient.update.mockResolvedValue(getRuleMock(getQueryRuleParams())); clients.detectionRulesClient.createCustomRule.mockResolvedValue(getRulesSchemaMock()); - clients.detectionRulesClient.legacyImportRule.mockResolvedValue(getRulesSchemaMock()); + clients.detectionRulesClient.importRule.mockResolvedValue(getRulesSchemaMock()); clients.actionsClient.getAll.mockResolvedValue([]); context.core.elasticsearch.client.asCurrentUser.search.mockResolvedValue( elasticsearchClientMock.createSuccessTransportRequestPromise(getBasicEmptySearchResponse()) @@ -90,7 +90,7 @@ describe('Import rules route', () => { describe('unhappy paths', () => { test('returns a 403 error object if ML Authz fails', async () => { - clients.detectionRulesClient.legacyImportRule.mockImplementationOnce(async () => { + clients.detectionRulesClient.importRule.mockImplementationOnce(async () => { throw new HttpAuthzError('mocked validation message'); }); @@ -219,7 +219,7 @@ describe('Import rules route', () => { describe('rule with existing rule_id', () => { test('returns with reported conflict if `overwrite` is set to `false`', async () => { clients.rulesClient.find.mockResolvedValue(getFindResultWithSingleHit()); // extant rule - clients.detectionRulesClient.legacyImportRule.mockRejectedValue({ + clients.detectionRulesClient.importRule.mockRejectedValue({ message: 'rule_id: "rule-1" already exists', statusCode: 409, }); @@ -437,7 +437,7 @@ describe('Import rules route', () => { }); test('returns with reported conflict if `overwrite` is set to `false`', async () => { - clients.detectionRulesClient.legacyImportRule.mockRejectedValueOnce({ + clients.detectionRulesClient.importRule.mockRejectedValueOnce({ message: 'rule_id: "rule-1" already exists', statusCode: 409, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts index 13299af6f4201..19d028bb9e666 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts @@ -17,7 +17,6 @@ const createDetectionRulesClientMock = () => { patchRule: jest.fn(), deleteRule: jest.fn(), upgradePrebuiltRule: jest.fn(), - legacyImportRule: jest.fn(), importRule: jest.fn(), importRules: jest.fn(), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts index cc3da7fb21f96..4300b17b3be80 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts @@ -17,10 +17,7 @@ import { getQueryRuleParams } from '../../../rule_schema/mocks'; import { createDetectionRulesClient } from './detection_rules_client'; import type { IDetectionRulesClient } from './detection_rules_client_interface'; import { getRuleByRuleId } from './methods/get_rule_by_rule_id'; -import { - getValidatedRuleToImportMock, - getValidatedRuleToImportWithSourceMock, -} from '../../../../../../common/api/detection_engine/rule_management/mocks'; +import { getValidatedRuleToImportMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; jest.mock('../../../../machine_learning/authz'); jest.mock('../../../../machine_learning/validation'); @@ -36,7 +33,7 @@ describe('DetectionRulesClient.importRule', () => { const allowMissingConnectorSecrets = true; const ruleToImport = { - ...getValidatedRuleToImportWithSourceMock(), + ...getValidatedRuleToImportMock(), tags: ['import-tag'], rule_id: 'rule-id', version: 1, @@ -74,7 +71,7 @@ describe('DetectionRulesClient.importRule', () => { immutable: ruleToImport.immutable, ruleId: ruleToImport.rule_id, version: ruleToImport.version, - ruleSource: ruleToImport.rule_source, + ruleSource: { type: 'internal' }, }), }), allowMissingConnectorSecrets, @@ -119,7 +116,7 @@ describe('DetectionRulesClient.importRule', () => { immutable: ruleToImport.immutable, ruleId: ruleToImport.rule_id, version: ruleToImport.version, - ruleSource: ruleToImport.rule_source, + ruleSource: { type: 'internal' }, }), }), id: existingRule.id, @@ -230,6 +227,39 @@ describe('DetectionRulesClient.importRule', () => { ); }); + it('preserves the passed "rule_source" and "immutable" values', async () => { + const rule = { + ...getValidatedRuleToImportMock(), + }; + + await detectionRulesClient.importRule({ + ruleToImport: rule, + overwriteRules: true, + overrideFields: { + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }, + allowMissingConnectorSecrets, + }); + + expect(rulesClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + immutable: true, + ruleSource: { + isCustomized: true, + type: 'external', + }, + }), + }), + }) + ); + }); + it('rejects when overwriteRules is false', async () => { (getRuleByRuleId as jest.Mock).mockResolvedValue(existingRule); await expect( @@ -250,12 +280,7 @@ describe('DetectionRulesClient.importRule', () => { it("always uses the existing rule's 'id' value", async () => { const rule = { ...getValidatedRuleToImportMock(), - immutable: true, id: 'some-id', - rule_source: { - type: 'external' as const, - is_customized: true, - }, }; await detectionRulesClient.importRule({ @@ -275,11 +300,7 @@ describe('DetectionRulesClient.importRule', () => { it("uses the existing rule's 'version' value if not unspecified", async () => { const rule = { ...getValidatedRuleToImportMock(), - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, + version: undefined, }; await detectionRulesClient.importRule({ @@ -303,12 +324,7 @@ describe('DetectionRulesClient.importRule', () => { it("uses the specified 'version' value", async () => { const rule = { ...getValidatedRuleToImportMock(), - immutable: true, version: 42, - rule_source: { - type: 'external' as const, - is_customized: true, - }, }; await detectionRulesClient.importRule({ @@ -338,16 +354,18 @@ describe('DetectionRulesClient.importRule', () => { it('preserves the passed "rule_source" and "immutable" values', async () => { const rule = { ...getValidatedRuleToImportMock(), - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, }; await detectionRulesClient.importRule({ ruleToImport: rule, overwriteRules: true, + overrideFields: { + immutable: true, + rule_source: { + type: 'external' as const, + is_customized: true, + }, + }, allowMissingConnectorSecrets, }); @@ -370,11 +388,6 @@ describe('DetectionRulesClient.importRule', () => { const rule = { ...getValidatedRuleToImportMock(), enabled: true, - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, }; await detectionRulesClient.importRule({ @@ -395,11 +408,6 @@ describe('DetectionRulesClient.importRule', () => { it('defaults defaultable values', async () => { const rule = { ...getValidatedRuleToImportMock(), - immutable: true, - rule_source: { - type: 'external' as const, - is_customized: true, - }, }; await detectionRulesClient.importRule({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts index 1ce48106d0ae1..d09729cf870ba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts @@ -14,11 +14,11 @@ import { getImportRulesSchemaMock } from '../../../../../../common/api/detection import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; import { ruleSourceImporterMock } from '../../../prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock'; import { createDetectionRulesClient } from './detection_rules_client'; -import { importRuleWithSource } from './methods/import_rule_with_source'; +import { importRule } from './methods/import_rule'; import { createRuleImportErrorObject } from '../import/errors'; import { checkRuleExceptionReferences } from '../import/check_rule_exception_references'; -jest.mock('./methods/import_rule_with_source'); +jest.mock('./methods/import_rule'); jest.mock('../import/check_rule_exception_references'); describe('detectionRulesClient.importRules', () => { @@ -35,7 +35,7 @@ describe('detectionRulesClient.importRules', () => { }); (checkRuleExceptionReferences as jest.Mock).mockReturnValue([[], []]); - (importRuleWithSource as jest.Mock).mockResolvedValue(getRulesSchemaMock()); + (importRule as jest.Mock).mockResolvedValue(getRulesSchemaMock()); ruleToImport = getImportRulesSchemaMock(); mockRuleSourceImporter = ruleSourceImporterMock.create(); @@ -61,7 +61,7 @@ describe('detectionRulesClient.importRules', () => { ruleId: 'rule-id', message: 'an error occurred', }); - (importRuleWithSource as jest.Mock).mockReset().mockRejectedValueOnce(importError); + (importRule as jest.Mock).mockReset().mockRejectedValueOnce(importError); const result = await subject.importRules({ allowMissingConnectorSecrets: false, @@ -75,7 +75,7 @@ describe('detectionRulesClient.importRules', () => { it('returns a generic error if rule import throws unexpectedly', async () => { const genericError = new Error('an unexpected error occurred'); - (importRuleWithSource as jest.Mock).mockReset().mockRejectedValueOnce(genericError); + (importRule as jest.Mock).mockReset().mockRejectedValueOnce(genericError); const result = await subject.importRules({ allowMissingConnectorSecrets: false, @@ -133,7 +133,7 @@ describe('detectionRulesClient.importRules', () => { ruleId: 'rule-id', message: 'an error occurred', }); - (importRuleWithSource as jest.Mock).mockReset().mockRejectedValueOnce(importError); + (importRule as jest.Mock).mockReset().mockRejectedValueOnce(importError); const result = await subject.importRules({ allowMissingConnectorSecrets: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.legacy_import_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.legacy_import_rule.test.ts deleted file mode 100644 index 5108729cc9cc8..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.legacy_import_rule.test.ts +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; -import type { ActionsClient } from '@kbn/actions-plugin/server'; - -import { savedObjectsClientMock } from '@kbn/core/server/mocks'; -import { - getCreateRulesSchemaMock, - getRulesSchemaMock, -} from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; -import { buildMlAuthz } from '../../../../machine_learning/authz'; -import { throwAuthzError } from '../../../../machine_learning/validation'; -import { getRuleMock } from '../../../routes/__mocks__/request_responses'; -import { getQueryRuleParams } from '../../../rule_schema/mocks'; -import { createDetectionRulesClient } from './detection_rules_client'; -import type { IDetectionRulesClient } from './detection_rules_client_interface'; -import { getRuleByRuleId } from './methods/get_rule_by_rule_id'; - -jest.mock('../../../../machine_learning/authz'); -jest.mock('../../../../machine_learning/validation'); - -jest.mock('./methods/get_rule_by_rule_id'); - -describe('DetectionRulesClient.legacyImportRule', () => { - let rulesClient: ReturnType; - let detectionRulesClient: IDetectionRulesClient; - - const mlAuthz = (buildMlAuthz as jest.Mock)(); - let actionsClient: jest.Mocked; - - const immutable = false as const; // Can only take value of false - const allowMissingConnectorSecrets = true; - const ruleToImport = { - ...getCreateRulesSchemaMock(), - tags: ['import-tag'], - rule_id: 'rule-id', - version: 1, - immutable, - }; - const existingRule = getRulesSchemaMock(); - existingRule.rule_id = ruleToImport.rule_id; - - beforeEach(() => { - rulesClient = rulesClientMock.create(); - rulesClient.create.mockResolvedValue(getRuleMock(getQueryRuleParams())); - rulesClient.update.mockResolvedValue(getRuleMock(getQueryRuleParams())); - const savedObjectsClient = savedObjectsClientMock.create(); - detectionRulesClient = createDetectionRulesClient({ - actionsClient, - rulesClient, - mlAuthz, - savedObjectsClient, - }); - }); - - it('calls rulesClient.create with the correct parameters when rule_id does not match an installed rule', async () => { - (getRuleByRuleId as jest.Mock).mockResolvedValueOnce(null); - await detectionRulesClient.legacyImportRule({ - ruleToImport, - overwriteRules: true, - allowMissingConnectorSecrets, - }); - - expect(rulesClient.create).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - name: ruleToImport.name, - tags: ruleToImport.tags, - params: expect.objectContaining({ - immutable, - ruleId: ruleToImport.rule_id, - version: ruleToImport.version, - }), - }), - allowMissingConnectorSecrets, - }) - ); - }); - - it('throws if mlAuth fails', async () => { - (throwAuthzError as jest.Mock).mockImplementationOnce(() => { - throw new Error('mocked MLAuth error'); - }); - - await expect( - detectionRulesClient.legacyImportRule({ - ruleToImport, - overwriteRules: true, - allowMissingConnectorSecrets, - }) - ).rejects.toThrow('mocked MLAuth error'); - - expect(rulesClient.create).not.toHaveBeenCalled(); - expect(rulesClient.update).not.toHaveBeenCalled(); - }); - - describe('when rule_id matches an installed rule', () => { - it('calls rulesClient.update with the correct parameters when overwriteRules is true', async () => { - (getRuleByRuleId as jest.Mock).mockResolvedValueOnce(existingRule); - - await detectionRulesClient.legacyImportRule({ - ruleToImport, - overwriteRules: true, - allowMissingConnectorSecrets, - }); - - expect(rulesClient.update).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - name: ruleToImport.name, - tags: ruleToImport.tags, - params: expect.objectContaining({ - index: ruleToImport.index, - description: ruleToImport.description, - }), - }), - id: existingRule.id, - }) - ); - }); - - /** - * Existing rule may have nullable fields set to a value (e.g. `timestamp_override` is set to `some.value`) but - * a rule to import doesn't have these fields set (e.g. `timestamp_override` is NOT present at all in the ndjson file). - * We expect the updated rule won't have such fields preserved (e.g. `timestamp_override` will be removed). - * - * Unit test is only able to check `updateRules()` receives a proper update object. - */ - it('ensures overwritten rule DOES NOT preserve fields missed in the imported rule when "overwriteRules" is "true" and matching rule found', async () => { - const existingRuleWithTimestampOverride = { - ...existingRule, - timestamp_override: '2020-01-01T00:00:00Z', - }; - (getRuleByRuleId as jest.Mock).mockResolvedValue(existingRuleWithTimestampOverride); - - await detectionRulesClient.legacyImportRule({ - ruleToImport: { - ...ruleToImport, - timestamp_override: undefined, - }, - overwriteRules: true, - allowMissingConnectorSecrets, - }); - - expect(rulesClient.create).not.toHaveBeenCalled(); - expect(rulesClient.update).toHaveBeenCalledWith( - expect.objectContaining({ - id: existingRule.id, - data: expect.not.objectContaining({ - timestamp_override: expect.anything(), - timestampOverride: expect.anything(), - }), - }) - ); - }); - - it('enables the rule if the imported rule has enabled: true', async () => { - const disabledExistingRule = { - ...existingRule, - enabled: false, - }; - (getRuleByRuleId as jest.Mock).mockResolvedValueOnce(disabledExistingRule); - - const rule = await detectionRulesClient.legacyImportRule({ - ruleToImport: { - ...ruleToImport, - enabled: true, - }, - overwriteRules: true, - allowMissingConnectorSecrets, - }); - - expect(rulesClient.create).not.toHaveBeenCalled(); - expect(rulesClient.update).toHaveBeenCalledWith( - expect.objectContaining({ - id: existingRule.id, - data: expect.not.objectContaining({ - enabled: expect.anything(), - }), - }) - ); - - expect(rule.enabled).toBe(true); - expect(rulesClient.enableRule).toHaveBeenCalledWith( - expect.objectContaining({ - id: existingRule.id, - }) - ); - }); - - it('disables the rule if the imported rule has enabled: false', async () => { - const enabledExistingRule = { - ...existingRule, - enabled: true, - }; - (getRuleByRuleId as jest.Mock).mockResolvedValueOnce(enabledExistingRule); - - const rule = await detectionRulesClient.legacyImportRule({ - ruleToImport: { - ...ruleToImport, - enabled: false, - }, - overwriteRules: true, - allowMissingConnectorSecrets, - }); - - expect(rulesClient.create).not.toHaveBeenCalled(); - expect(rulesClient.update).toHaveBeenCalledWith( - expect.objectContaining({ - id: existingRule.id, - data: expect.not.objectContaining({ - enabled: expect.anything(), - }), - }) - ); - - expect(rule.enabled).toBe(false); - expect(rulesClient.disableRule).toHaveBeenCalledWith( - expect.objectContaining({ - id: existingRule.id, - }) - ); - }); - - it('rejects when overwriteRules is false', async () => { - (getRuleByRuleId as jest.Mock).mockResolvedValue(existingRule); - await expect( - detectionRulesClient.legacyImportRule({ - ruleToImport, - overwriteRules: false, - allowMissingConnectorSecrets, - }) - ).rejects.toMatchObject({ - error: { - status_code: 409, - message: `rule_id: "${ruleToImport.rule_id}" already exists`, - }, - rule_id: ruleToImport.rule_id, - }); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts index 81d93dac94597..fb6f5c4a03c1f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.ts @@ -21,7 +21,6 @@ import type { IDetectionRulesClient, ImportRuleArgs, ImportRulesArgs, - LegacyImportRuleArgs, PatchRuleArgs, UpdateRuleArgs, UpgradePrebuiltRuleArgs, @@ -32,7 +31,6 @@ import { importRule } from './methods/import_rule'; import { patchRule } from './methods/patch_rule'; import { updateRule } from './methods/update_rule'; import { upgradePrebuiltRule } from './methods/upgrade_prebuilt_rule'; -import { importRuleWithSource } from './methods/import_rule_with_source'; import { importRules } from './methods/import_rules'; interface DetectionRulesClientParams { @@ -125,21 +123,9 @@ export const createDetectionRulesClient = ({ }); }, - async legacyImportRule(args: LegacyImportRuleArgs): Promise { - return withSecuritySpan('DetectionRulesClient.legacyImportRule', async () => { - return importRule({ - actionsClient, - rulesClient, - importRulePayload: args, - mlAuthz, - prebuiltRuleAssetClient, - }); - }); - }, - async importRule(args: ImportRuleArgs): Promise { return withSecuritySpan('DetectionRulesClient.importRule', async () => { - return importRuleWithSource({ + return importRule({ actionsClient, rulesClient, importRulePayload: args, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts index 2aa57ed5b95d0..a2f181f607468 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts @@ -11,7 +11,6 @@ import type { RulePatchProps, RuleObjectId, RuleResponse, - ValidatedRuleToImport, RuleToImport, RuleSource, } from '../../../../../../common/api/detection_engine'; @@ -26,7 +25,6 @@ export interface IDetectionRulesClient { patchRule: (args: PatchRuleArgs) => Promise; deleteRule: (args: DeleteRuleArgs) => Promise; upgradePrebuiltRule: (args: UpgradePrebuiltRuleArgs) => Promise; - legacyImportRule: (args: LegacyImportRuleArgs) => Promise; importRule: (args: ImportRuleArgs) => Promise; importRules: (args: ImportRulesArgs) => Promise>; } @@ -55,14 +53,9 @@ export interface UpgradePrebuiltRuleArgs { ruleAsset: PrebuiltRuleAsset; } -export interface LegacyImportRuleArgs { - ruleToImport: RuleToImport; - overwriteRules?: boolean; - allowMissingConnectorSecrets?: boolean; -} - export interface ImportRuleArgs { - ruleToImport: ValidatedRuleToImport & { rule_source: RuleSource; immutable: boolean }; + ruleToImport: RuleToImport; + overrideFields?: { rule_source: RuleSource; immutable: boolean }; overwriteRules?: boolean; allowMissingConnectorSecrets?: boolean; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts index c60f7c0098655..dd57e66c41a64 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule.ts @@ -11,20 +11,20 @@ import type { ActionsClient } from '@kbn/actions-plugin/server'; import type { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; import type { MlAuthz } from '../../../../../machine_learning/authz'; import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; -import { createBulkErrorObject } from '../../../../routes/utils'; import { convertAlertingRuleToRuleResponse } from '../converters/convert_alerting_rule_to_rule_response'; import { convertRuleResponseToAlertingRule } from '../converters/convert_rule_response_to_alerting_rule'; -import type { LegacyImportRuleArgs } from '../detection_rules_client_interface'; +import type { ImportRuleArgs } from '../detection_rules_client_interface'; import { applyRuleUpdate } from '../mergers/apply_rule_update'; import { validateMlAuth, toggleRuleEnabledOnUpdate } from '../utils'; import { createRule } from './create_rule'; import { getRuleByRuleId } from './get_rule_by_rule_id'; +import { createRuleImportErrorObject } from '../../import/errors'; interface ImportRuleOptions { actionsClient: ActionsClient; rulesClient: RulesClient; prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; - importRulePayload: LegacyImportRuleArgs; + importRulePayload: ImportRuleArgs; mlAuthz: MlAuthz; } @@ -35,9 +35,10 @@ export const importRule = async ({ prebuiltRuleAssetClient, mlAuthz, }: ImportRuleOptions): Promise => { - const { ruleToImport, overwriteRules, allowMissingConnectorSecrets } = importRulePayload; - // For backwards compatibility in this legacy path, immutable is always false. - const rule = { ...ruleToImport, immutable: false }; + const { ruleToImport, overwriteRules, overrideFields, allowMissingConnectorSecrets } = + importRulePayload; + // For backwards compatibility, immutable is false by default + const rule = { ...ruleToImport, immutable: false, ...overrideFields }; await validateMlAuth(mlAuthz, ruleToImport.type); @@ -47,19 +48,21 @@ export const importRule = async ({ }); if (existingRule && !overwriteRules) { - throw createBulkErrorObject({ + throw createRuleImportErrorObject({ ruleId: existingRule.rule_id, - statusCode: 409, + type: 'conflict', message: `rule_id: "${existingRule.rule_id}" already exists`, }); } if (existingRule && overwriteRules) { - const ruleWithUpdates = await applyRuleUpdate({ + let ruleWithUpdates = await applyRuleUpdate({ prebuiltRuleAssetClient, existingRule, ruleUpdate: rule, }); + // applyRuleUpdate prefers the existing rule's values for `rule_source` and `immutable`, but we want to use the importing rule's calculated values + ruleWithUpdates = { ...ruleWithUpdates, ...overrideFields }; const updatedRule = await rulesClient.update({ id: existingRule.id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts deleted file mode 100644 index 408485b2952dc..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rule_with_source.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { RulesClient } from '@kbn/alerting-plugin/server'; -import type { ActionsClient } from '@kbn/actions-plugin/server'; -import { ruleTypeMappings } from '@kbn/securitysolution-rules'; - -import type { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; -import type { MlAuthz } from '../../../../../machine_learning/authz'; -import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; -import { convertAlertingRuleToRuleResponse } from '../converters/convert_alerting_rule_to_rule_response'; -import { convertRuleResponseToAlertingRule } from '../converters/convert_rule_response_to_alerting_rule'; -import type { ImportRuleArgs } from '../detection_rules_client_interface'; -import { applyRuleUpdate } from '../mergers/apply_rule_update'; -import { applyRuleDefaults } from '../mergers/apply_rule_defaults'; -import { validateMlAuth, toggleRuleEnabledOnUpdate } from '../utils'; -import { getRuleByRuleId } from './get_rule_by_rule_id'; -import { SERVER_APP_ID } from '../../../../../../../common'; -import { createRuleImportErrorObject } from '../../import/errors'; - -interface ImportRuleOptions { - actionsClient: ActionsClient; - rulesClient: RulesClient; - prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; - importRulePayload: ImportRuleArgs; - mlAuthz: MlAuthz; -} - -/** - * Imports a rule with pre-calculated `rule_source` (and `immutable`) fields. - * Once prebuilt rule customization epic is complete, this function can be used - * to import all rules. Until then, this function should only be used when the feature flag is enabled. - * - */ -export const importRuleWithSource = async ({ - actionsClient, - rulesClient, - importRulePayload, - prebuiltRuleAssetClient, - mlAuthz, -}: ImportRuleOptions): Promise => { - const { ruleToImport, overwriteRules, allowMissingConnectorSecrets } = importRulePayload; - - await validateMlAuth(mlAuthz, ruleToImport.type); - - const existingRule = await getRuleByRuleId({ - rulesClient, - ruleId: ruleToImport.rule_id, - }); - - if (existingRule && !overwriteRules) { - throw createRuleImportErrorObject({ - ruleId: existingRule.rule_id, - type: 'conflict', - message: `rule_id: "${existingRule.rule_id}" already exists`, - }); - } - - if (existingRule && overwriteRules) { - const ruleWithUpdates = await applyRuleUpdate({ - prebuiltRuleAssetClient, - existingRule, - ruleUpdate: ruleToImport, - }); - // applyRuleUpdate prefers the existing rule's values for `rule_source` and `immutable`, but we want to use the importing rule's calculated values - ruleWithUpdates.immutable = ruleToImport.immutable; - ruleWithUpdates.rule_source = ruleToImport.rule_source; - - const updatedRule = await rulesClient.update({ - id: existingRule.id, - data: convertRuleResponseToAlertingRule(ruleWithUpdates, actionsClient), - }); - - // We strip `enabled` from the rule object to use in the rules client and need to enable it separately if user has enabled the updated rule - const { enabled } = await toggleRuleEnabledOnUpdate(rulesClient, existingRule, ruleWithUpdates); - - return convertAlertingRuleToRuleResponse({ ...updatedRule, enabled }); - } - - const ruleWithDefaults = applyRuleDefaults(ruleToImport); - - const payload = { - ...convertRuleResponseToAlertingRule(ruleWithDefaults, actionsClient), - alertTypeId: ruleTypeMappings[ruleToImport.type], - consumer: SERVER_APP_ID, - enabled: ruleToImport.enabled ?? false, - }; - - const createdRule = await rulesClient.create({ - data: payload, - options: { id: undefined }, - allowMissingConnectorSecrets, - }); - - return convertAlertingRuleToRuleResponse(createdRule); -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts index 42ef6c621e93e..cf6053b73b510 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts @@ -79,10 +79,9 @@ export const importRules = async ({ const importedRule = await detectionRulesClient.importRule({ ruleToImport: { ...rule, - rule_source: ruleSource, - immutable, exceptions_list: [...exceptions], }, + overrideFields: { rule_source: ruleSource, immutable }, overwriteRules, allowMissingConnectorSecrets, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts index c240a4f82f26c..eb2b25d6de850 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts @@ -12,7 +12,7 @@ import { getRulesSchemaMock } from '../../../../../../common/api/detection_engin import { requestContextMock } from '../../../routes/__mocks__'; import { importRules } from './import_rules_utils'; -import { createBulkErrorObject } from '../../../routes/utils'; +import { createRuleImportErrorObject } from './errors'; describe('importRules', () => { const { clients, context } = requestContextMock.createTools(); @@ -59,10 +59,10 @@ describe('importRules', () => { }); it('returns 409 error if DetectionRulesClient throws with 409 - existing rule', async () => { - clients.detectionRulesClient.legacyImportRule.mockImplementationOnce(async () => { - throw createBulkErrorObject({ + clients.detectionRulesClient.importRule.mockImplementationOnce(async () => { + throw createRuleImportErrorObject({ ruleId: ruleToImport.rule_id, - statusCode: 409, + type: 'conflict', message: `rule_id: "${ruleToImport.rule_id}" already exists`, }); }); @@ -88,7 +88,7 @@ describe('importRules', () => { }); it('creates rule if no matching existing rule found', async () => { - clients.detectionRulesClient.legacyImportRule.mockResolvedValue({ + clients.detectionRulesClient.importRule.mockResolvedValue({ ...getRulesSchemaMock(), rule_id: ruleToImport.rule_id, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 0b1fb6f20036f..39b6f89905357 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -14,6 +14,7 @@ import { checkRuleExceptionReferences } from './check_rule_exception_references' import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { getReferencedExceptionLists } from './gather_referenced_exceptions'; import type { RuleFromImportStream } from './utils'; +import { isRuleConflictError, isRuleImportError } from './errors'; /** * Takes rules to be imported and either creates or updates rules @@ -109,7 +110,7 @@ export const importRules = async ({ importRuleResponse = [...importRuleResponse, ...exceptionBulkErrors]; - const importedRule = await detectionRulesClient.legacyImportRule({ + const importedRule = await detectionRulesClient.importRule({ ruleToImport: { ...parsedRule, exceptions_list: [...exceptions], @@ -124,6 +125,17 @@ export const importRules = async ({ }); } catch (err) { const { error, statusCode, message } = err; + if (isRuleImportError(err)) { + resolve( + createBulkErrorObject({ + message: err.error.message, + statusCode: isRuleConflictError(err) ? 409 : 400, + ruleId: err.error.ruleId, + }) + ); + return null; + } + resolve( createBulkErrorObject({ ruleId: parsedRule.rule_id, From 2b497dabab3654cb5ffd31c58efce6ea27f93234 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 12:56:03 -0500 Subject: [PATCH 092/120] Reverts some previous changes to `internalRuleToAPIResponse` We had moved to using the more general `convertObjectKeysToSnakeCase` in this transformation method, but it ended up being more than we needed with some complicated downstream changes. This reverts those changes to just what is needed (namely: the changes in the child converters and in `commonParamsCamelToSnake`). This partially reverts both 17f68a39d28f217b59df5f9c867b94702af7dcb0 and b3944dca227dcf0576a7f3a3844be36fa6f87bbd. --- .../detection_engine/routes/__mocks__/utils.ts | 2 +- .../converters/internal_rule_to_api_response.ts | 15 ++++++++------- .../logic/export/get_export_all.test.ts | 10 +++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts index 3d4a8105dc151..687bf91655e2a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts @@ -81,7 +81,7 @@ export const getOutputRuleAlertForRest = (): RuleResponse => ({ }, ], meta: { - some_meta: 'someField', + someMeta: 'someField', }, timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts index d364066be56b7..336cd8fae0405 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts @@ -6,24 +6,25 @@ */ import type { ResolvedSanitizedRule, SanitizedRule } from '@kbn/alerting-plugin/common'; +import type { RequiredOptional } from '@kbn/zod-helpers'; import type { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; import { transformAlertToRuleAction, transformAlertToRuleSystemAction, } from '../../../../../../../common/detection_engine/transform_actions'; import { createRuleExecutionSummary } from '../../../../rule_monitoring'; -import { type RuleParams } from '../../../../rule_schema'; +import type { RuleParams } from '../../../../rule_schema'; import { transformFromAlertThrottle, transformToActionFrequency, } from '../../../normalization/rule_actions'; import { typeSpecificCamelToSnake } from './type_specific_camel_to_snake'; +import { commonParamsCamelToSnake } from './common_params_camel_to_snake'; import { normalizeRuleParams } from './normalize_rule_params'; -import { convertObjectKeysToSnakeCase } from '../../../../../../utils/object_case_converters'; export const internalRuleToAPIResponse = ( rule: SanitizedRule | ResolvedSanitizedRule -): RuleResponse => { +): RequiredOptional => { const executionSummary = createRuleExecutionSummary(rule); const isResolvedRule = (obj: unknown): obj is ResolvedSanitizedRule => { @@ -39,8 +40,6 @@ export const internalRuleToAPIResponse = ( return transformedAction; }); const normalizedRuleParams = normalizeRuleParams(rule.params); - const commonRuleParams = convertObjectKeysToSnakeCase(normalizedRuleParams); - const typeSpecificRuleParams = typeSpecificCamelToSnake(rule.params); return { // saved object properties @@ -58,8 +57,10 @@ export const internalRuleToAPIResponse = ( interval: rule.schedule.interval, enabled: rule.enabled, revision: rule.revision, - ...commonRuleParams, - ...typeSpecificRuleParams, + // Security solution shared rule params + ...commonParamsCamelToSnake(normalizedRuleParams), + // Type specific security solution rule params + ...typeSpecificCamelToSnake(rule.params), // Actions throttle: undefined, actions: [...actions, ...(systemActions ?? [])], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts index f60eedd0ee8b8..382df4bfa5ffc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts @@ -70,7 +70,7 @@ describe('getExportAll', () => { ...alert.params, filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], threat: getThreatMock(), - meta: { some_meta: 'someField' }, + meta: { someMeta: 'someField' }, timelineId: 'some-timeline-id', timelineTitle: 'some-timeline-title', }; @@ -121,7 +121,7 @@ describe('getExportAll', () => { setup: '', timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', - meta: { some_meta: 'someField' }, + meta: { someMeta: 'someField' }, severity: 'high', severity_mapping: [], updated_by: 'elastic', @@ -200,7 +200,7 @@ describe('getExportAll', () => { ...alert.params, filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], threat: getThreatMock(), - meta: { some_meta: 'someField' }, + meta: { someMeta: 'someField' }, timelineId: 'some-timeline-id', timelineTitle: 'some-timeline-title', }; @@ -303,7 +303,7 @@ describe('getExportAll', () => { setup: '', timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', - meta: { some_meta: 'someField' }, + meta: { someMeta: 'someField' }, severity: 'high', severity_mapping: [], updated_by: 'elastic', @@ -375,7 +375,7 @@ describe('getExportAll', () => { ...alert.params, filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], threat: getThreatMock(), - meta: { some_meta: 'someField' }, + meta: { someMeta: 'someField' }, timelineId: 'some-timeline-id', timelineTitle: 'some-timeline-title', }; From 41bcee3ab5794ef29630116dbd0973d9acf1f29a Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 13:52:38 -0500 Subject: [PATCH 093/120] Move parser error handling to the import route level This handles parser errors at the route level, as opposed to passing them down and then expecting error messages to be returned (as we had previously been doing). --- .../api/rules/import_rules/route.ts | 36 +++++++++++++------ .../logic/import/import_rules.test.ts | 29 --------------- .../logic/import/import_rules.ts | 18 +++------- .../logic/import/import_rules_utils.test.ts | 20 ----------- .../logic/import/import_rules_utils.ts | 16 ++------- 5 files changed, 32 insertions(+), 87 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 5701663eb5925..2d13cf3b6403d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -8,7 +8,7 @@ import { schema } from '@kbn/config-schema'; import type { IKibanaResponse } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import { chunk } from 'lodash/fp'; +import { chunk, partition } from 'lodash/fp'; import { extname } from 'path'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { @@ -18,8 +18,13 @@ import { import { DETECTION_ENGINE_RULES_URL } from '../../../../../../../common/constants'; import type { ConfigType } from '../../../../../../config'; import type { HapiReadableStream, SecuritySolutionPluginRouter } from '../../../../../../types'; -import type { BulkError, ImportRuleResponse } from '../../../../routes/utils'; -import { buildSiemResponse, isBulkError, isImportRegular } from '../../../../routes/utils'; +import type { ImportRuleResponse } from '../../../../routes/utils'; +import { + buildSiemResponse, + createBulkErrorObject, + isBulkError, + isImportRegular, +} from '../../../../routes/utils'; import { createRuleSourceImporter } from '../../../../prebuilt_rules/logic/rule_source_importer'; import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; @@ -27,6 +32,7 @@ import { importRules } from '../../../logic/import/import_rules'; import { importRules as legacyImportRules } from '../../../logic/import/import_rules_utils'; import { createPromiseFromRuleImportStream } from '../../../logic/import/create_promise_from_rule_import_stream'; import { importRuleExceptions } from '../../../logic/import/import_rule_exceptions'; +import { isRuleToImport } from '../../../logic/import/utils'; import { getTupleDuplicateErrorsAndUniqueRules, migrateLegacyActionsIds, @@ -139,7 +145,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C // rulesWithMigratedActions: Is returned only in case connectors were exported from different namespace and the // original rules actions' ids were replaced with new destinationIds - const parsedRules = actionConnectorErrors.length + const parsedRuleStream = actionConnectorErrors.length ? [] : rulesWithMigratedActions || migratedParsedObjectsWithoutDuplicateErrors; @@ -149,13 +155,14 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C prebuiltRuleAssetsClient: createPrebuiltRuleAssetsClient(savedObjectsClient), }); - const chunkParseObjects = chunk(CHUNK_PARSED_OBJECT_SIZE, parsedRules); + const [parsedRules, parsedRuleErrors] = partition(isRuleToImport, parsedRuleStream); + const ruleChunks = chunk(CHUNK_PARSED_OBJECT_SIZE, parsedRules); let importRuleResponse: ImportRuleResponse[] = []; if (prebuiltRulesCustomizationEnabled) { importRuleResponse = await importRules({ - ruleChunks: chunkParseObjects, + ruleChunks, rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], overwriteRules: request.query.overwrite, allowMissingConnectorSecrets: !!actionConnectors.length, @@ -164,7 +171,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C }); } else { importRuleResponse = await legacyImportRules({ - ruleChunks: chunkParseObjects, + ruleChunks, rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], overwriteRules: request.query.overwrite, allowMissingConnectorSecrets: !!actionConnectors.length, @@ -173,7 +180,15 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C }); } - const errorsResp = importRuleResponse.filter((resp) => isBulkError(resp)) as BulkError[]; + const parseErrors = parsedRuleErrors.map((error) => + createBulkErrorObject({ + statusCode: 400, + message: error.message, + }) + ); + const importErrors = importRuleResponse.filter(isBulkError); + const errors = [...parseErrors, ...importErrors]; + const successes = importRuleResponse.filter((resp) => { if (isImportRegular(resp)) { return resp.status_code === 200; @@ -181,11 +196,12 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C return false; } }); + const importedRules: ImportRulesResponse = { - success: errorsResp.length === 0, + success: errors.length === 0, success_count: successes.length, rules_count: rules.length, - errors: errorsResp, + errors, exceptions_errors: exceptionsErrors, exceptions_success: exceptionsSuccess, exceptions_success_count: exceptionsSuccessCount, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts index 4836d263256ce..ee47626504e05 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts @@ -41,35 +41,6 @@ describe('importRules', () => { expect(result).toEqual([]); }); - it('returns 400 errors if import was attempted on errored rules', async () => { - const rules = [new Error('error parsing'), new Error('error parsing')]; - - const result = await importRules({ - ruleChunks: [rules], - rulesResponseAcc: [], - overwriteRules: false, - detectionRulesClient, - ruleSourceImporter: mockRuleSourceImporter, - }); - - expect(result).toEqual([ - { - error: { - message: 'error parsing', - status_code: 400, - }, - rule_id: '(unknown id)', - }, - { - error: { - message: 'error parsing', - status_code: 400, - }, - rule_id: '(unknown id)', - }, - ]); - }); - it('returns 400 errors if client import returns generic errors', async () => { detectionRulesClient.importRules.mockResolvedValueOnce([ createRuleImportErrorObject({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts index 4dbadee0f1088..9d6c59892c9e9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts @@ -5,13 +5,11 @@ * 2.0. */ -import { partition } from 'lodash'; - +import type { RuleToImport } from '../../../../../../common/api/detection_engine'; import { type ImportRuleResponse, createBulkErrorObject } from '../../../routes/utils'; import type { IRuleSourceImporter } from '../../../prebuilt_rules/logic/rule_source_importer'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { isRuleConflictError, isRuleImportError } from './errors'; -import { isRuleToImport, type RuleFromImportStream } from './utils'; /** * Takes a stream of rules to be imported and either creates or updates rules @@ -35,7 +33,7 @@ export const importRules = async ({ ruleSourceImporter, allowMissingConnectorSecrets, }: { - ruleChunks: RuleFromImportStream[][]; + ruleChunks: RuleToImport[][]; rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; @@ -51,8 +49,7 @@ export const importRules = async ({ } while (ruleChunks.length) { - const ruleChunk = ruleChunks.shift() ?? []; - const [rules, errors] = partition(ruleChunk, isRuleToImport); + const rules = ruleChunks.shift() ?? []; const importedRulesResponse = await detectionRulesClient.importRules({ allowMissingConnectorSecrets, @@ -61,13 +58,6 @@ export const importRules = async ({ rules, }); - const genericErrors = errors.map((error) => - createBulkErrorObject({ - statusCode: 400, - message: error.message, - }) - ); - const importResponses = importedRulesResponse.map((rule) => { if (isRuleImportError(rule)) { return createBulkErrorObject({ @@ -83,7 +73,7 @@ export const importRules = async ({ }; }); - response = [...response, ...genericErrors, ...importResponses]; + response = [...response, ...importResponses]; } return response; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts index eb2b25d6de850..04951156d716a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts @@ -38,26 +38,6 @@ describe('importRules', () => { expect(result).toEqual([]); }); - it('returns 400 error if "ruleChunks" includes Error', async () => { - const result = await importRules({ - ruleChunks: [[new Error('error importing')]], - rulesResponseAcc: [], - overwriteRules: false, - detectionRulesClient: context.securitySolution.getDetectionRulesClient(), - savedObjectsClient, - }); - - expect(result).toEqual([ - { - error: { - message: 'error importing', - status_code: 400, - }, - rule_id: '(unknown id)', - }, - ]); - }); - it('returns 409 error if DetectionRulesClient throws with 409 - existing rule', async () => { clients.detectionRulesClient.importRule.mockImplementationOnce(async () => { throw createRuleImportErrorObject({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 39b6f89905357..d5666fe11c9e5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -8,12 +8,12 @@ import { i18n } from '@kbn/i18n'; import type { SavedObjectsClientContract } from '@kbn/core/server'; +import type { RuleToImport } from '../../../../../../common/api/detection_engine'; import type { ImportRuleResponse } from '../../../routes/utils'; import { createBulkErrorObject } from '../../../routes/utils'; import { checkRuleExceptionReferences } from './check_rule_exception_references'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { getReferencedExceptionLists } from './gather_referenced_exceptions'; -import type { RuleFromImportStream } from './utils'; import { isRuleConflictError, isRuleImportError } from './errors'; /** @@ -38,7 +38,7 @@ export const importRules = async ({ allowMissingConnectorSecrets, savedObjectsClient, }: { - ruleChunks: RuleFromImportStream[][]; + ruleChunks: RuleToImport[][]; rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; @@ -63,18 +63,6 @@ export const importRules = async ({ batchParseObjects.reduce>>((accum, parsedRule) => { const importsWorkerPromise = new Promise(async (resolve, reject) => { try { - if (parsedRule instanceof Error) { - // If the JSON object had a validation or parse error then we return - // early with the error and an (unknown) for the ruleId - resolve( - createBulkErrorObject({ - statusCode: 400, - message: parsedRule.message, - }) - ); - return null; - } - if (parsedRule.immutable) { resolve( createBulkErrorObject({ From 12569472592725ab73b5301e466234cf053549eb Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 14:07:33 -0500 Subject: [PATCH 094/120] Make importRules functions more pure This does away with the receiving, modifying, and returning of the `rulesResponseAcc` parameter from both the new and old import paths. This logic originally existed in the `import_rules_utils` interface, and was cargo-culted to the new code path(s) incorrectly by me. It's not necessary, so we simply do away with the complexity here. --- .../api/rules/import_rules/route.ts | 9 ++++++--- .../logic/import/import_rules.test.ts | 5 ----- .../logic/import/import_rules.ts | 13 +++---------- .../logic/import/import_rules_utils.test.ts | 4 ---- .../logic/import/import_rules_utils.ts | 19 ++++++------------- 5 files changed, 15 insertions(+), 35 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 2d13cf3b6403d..98f91456c5993 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -163,7 +163,6 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C if (prebuiltRulesCustomizationEnabled) { importRuleResponse = await importRules({ ruleChunks, - rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], overwriteRules: request.query.overwrite, allowMissingConnectorSecrets: !!actionConnectors.length, ruleSourceImporter, @@ -172,7 +171,6 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C } else { importRuleResponse = await legacyImportRules({ ruleChunks, - rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], overwriteRules: request.query.overwrite, allowMissingConnectorSecrets: !!actionConnectors.length, detectionRulesClient, @@ -187,7 +185,12 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C }) ); const importErrors = importRuleResponse.filter(isBulkError); - const errors = [...parseErrors, ...importErrors]; + const errors = [ + ...parseErrors, + ...actionConnectorErrors, + ...duplicateIdErrors, + ...importErrors, + ]; const successes = importRuleResponse.filter((resp) => { if (isImportRegular(resp)) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts index ee47626504e05..ae32cb7ba8ad2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts @@ -32,7 +32,6 @@ describe('importRules', () => { it('returns an empty rules response if no rules to import', async () => { const result = await importRules({ ruleChunks: [], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, ruleSourceImporter: mockRuleSourceImporter, @@ -55,7 +54,6 @@ describe('importRules', () => { const result = await importRules({ ruleChunks: [[ruleToImport]], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, ruleSourceImporter: mockRuleSourceImporter, @@ -95,7 +93,6 @@ describe('importRules', () => { const result = await importRules({ ruleChunks: [[ruleToImport]], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, ruleSourceImporter: mockRuleSourceImporter, @@ -137,7 +134,6 @@ describe('importRules', () => { const result = await importRules({ ruleChunks: [[ruleToImport]], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, ruleSourceImporter: mockRuleSourceImporter, @@ -172,7 +168,6 @@ describe('importRules', () => { const result = await importRules({ ruleChunks: [[ruleToImport]], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient, ruleSourceImporter: mockRuleSourceImporter, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts index 9d6c59892c9e9..325f1d037a8d8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts @@ -14,33 +14,26 @@ import { isRuleConflictError, isRuleImportError } from './errors'; /** * Takes a stream of rules to be imported and either creates or updates rules * based on user overwrite preferences - * @param ruleChunks {array} - rules being imported - * @param rulesResponseAcc {array} - the accumulation of success and - * error messages gathered through the rules import logic - * @param mlAuthz {object} + * @param ruleChunks {@link RuleToImport} - rules being imported * @param overwriteRules {boolean} - whether to overwrite existing rules * with imported rules if their rule_id matches * @param detectionRulesClient {object} - * @param existingLists {object} - all exception lists referenced by - * rules that were found to exist * @returns {Promise} an array of error and success messages from import */ export const importRules = async ({ ruleChunks, - rulesResponseAcc, overwriteRules, detectionRulesClient, ruleSourceImporter, allowMissingConnectorSecrets, }: { ruleChunks: RuleToImport[][]; - rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; ruleSourceImporter: IRuleSourceImporter; allowMissingConnectorSecrets?: boolean; }) => { - let response: ImportRuleResponse[] = [...rulesResponseAcc]; + const response: ImportRuleResponse[] = []; // If we had 100% errors and no successful rule could be imported we still have to output an error. // otherwise we would output we are success importing 0 rules. @@ -73,7 +66,7 @@ export const importRules = async ({ }; }); - response = [...response, ...importResponses]; + response.push(...importResponses); } return response; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts index 04951156d716a..6b0b3ce668c36 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts @@ -29,7 +29,6 @@ describe('importRules', () => { it('returns an empty rules response if no rules to import', async () => { const result = await importRules({ ruleChunks: [], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), savedObjectsClient, @@ -50,7 +49,6 @@ describe('importRules', () => { const ruleChunk = [ruleToImport]; const result = await importRules({ ruleChunks: [ruleChunk], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), savedObjectsClient, @@ -76,7 +74,6 @@ describe('importRules', () => { const ruleChunk = [ruleToImport]; const result = await importRules({ ruleChunks: [ruleChunk], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), savedObjectsClient, @@ -93,7 +90,6 @@ describe('importRules', () => { }; const result = await importRules({ ruleChunks: [[prebuiltRuleToImport]], - rulesResponseAcc: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), savedObjectsClient, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index d5666fe11c9e5..cefe8a0dc4744 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -19,38 +19,31 @@ import { isRuleConflictError, isRuleImportError } from './errors'; /** * Takes rules to be imported and either creates or updates rules * based on user overwrite preferences - * @param ruleChunks {array} - rules being imported - * @param rulesResponseAcc {array} - the accumulation of success and - * error messages gathered through the rules import logic - * @param mlAuthz {object} + * @param ruleChunks {@link RuleToImport} - rules being imported * @param overwriteRules {boolean} - whether to overwrite existing rules * with imported rules if their rule_id matches * @param detectionRulesClient {object} - * @param existingLists {object} - all exception lists referenced by - * rules that were found to exist * @returns {Promise} an array of error and success messages from import */ export const importRules = async ({ ruleChunks, - rulesResponseAcc, overwriteRules, detectionRulesClient, allowMissingConnectorSecrets, savedObjectsClient, }: { ruleChunks: RuleToImport[][]; - rulesResponseAcc: ImportRuleResponse[]; overwriteRules: boolean; detectionRulesClient: IDetectionRulesClient; allowMissingConnectorSecrets?: boolean; savedObjectsClient: SavedObjectsClientContract; }) => { - let importRuleResponse: ImportRuleResponse[] = [...rulesResponseAcc]; + const response: ImportRuleResponse[] = []; // If we had 100% errors and no successful rule could be imported we still have to output an error. // otherwise we would output we are success importing 0 rules. if (ruleChunks.length === 0) { - return importRuleResponse; + return response; } while (ruleChunks.length) { @@ -96,7 +89,7 @@ export const importRules = async ({ }) ); - importRuleResponse = [...importRuleResponse, ...exceptionBulkErrors]; + response.push(...exceptionBulkErrors); const importedRule = await detectionRulesClient.importRule({ ruleToImport: { @@ -139,8 +132,8 @@ export const importRules = async ({ return [...accum, importsWorkerPromise]; }, []) ); - importRuleResponse = [...importRuleResponse, ...newImportRuleResponse]; + response.push(...newImportRuleResponse); } - return importRuleResponse; + return response; }; From 5a685e8dc31b99594bb7a000af2225ef698bfd87 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 14:26:30 -0500 Subject: [PATCH 095/120] Mark old importRules function as legacy This requires some eslint declarations since we don't allow importing from '*legacy*' by default, but I think that's a reasonable tradeoff here. I also marked the old path as deprecated to further clarify/point toward the new function. --- .../api/rules/import_rules/route.ts | 5 +++-- ...s_utils.test.ts => import_rules_legacy.test.ts} | 14 ++++++++------ ...mport_rules_utils.ts => import_rules_legacy.ts} | 6 ++++-- 3 files changed, 15 insertions(+), 10 deletions(-) rename x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/{import_rules_utils.test.ts => import_rules_legacy.test.ts} (91%) rename x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/{import_rules_utils.ts => import_rules_legacy.ts} (97%) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 98f91456c5993..2bb2224b3e70d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -29,7 +29,8 @@ import { createRuleSourceImporter } from '../../../../prebuilt_rules/logic/rule_ import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; import { importRules } from '../../../logic/import/import_rules'; -import { importRules as legacyImportRules } from '../../../logic/import/import_rules_utils'; +// eslint-disable-next-line no-restricted-imports +import { importRulesLegacy } from '../../../logic/import/import_rules_legacy'; import { createPromiseFromRuleImportStream } from '../../../logic/import/create_promise_from_rule_import_stream'; import { importRuleExceptions } from '../../../logic/import/import_rule_exceptions'; import { isRuleToImport } from '../../../logic/import/utils'; @@ -169,7 +170,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C detectionRulesClient, }); } else { - importRuleResponse = await legacyImportRules({ + importRuleResponse = await importRulesLegacy({ ruleChunks, overwriteRules: request.query.overwrite, allowMissingConnectorSecrets: !!actionConnectors.length, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.test.ts similarity index 91% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.test.ts index 6b0b3ce668c36..0d8fe8118471b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.test.ts @@ -11,10 +11,12 @@ import { getImportRulesSchemaMock } from '../../../../../../common/api/detection import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; import { requestContextMock } from '../../../routes/__mocks__'; -import { importRules } from './import_rules_utils'; import { createRuleImportErrorObject } from './errors'; -describe('importRules', () => { +// eslint-disable-next-line no-restricted-imports +import { importRulesLegacy } from './import_rules_legacy'; + +describe('importRulesLegacy', () => { const { clients, context } = requestContextMock.createTools(); const ruleToImport = getImportRulesSchemaMock(); @@ -27,7 +29,7 @@ describe('importRules', () => { }); it('returns an empty rules response if no rules to import', async () => { - const result = await importRules({ + const result = await importRulesLegacy({ ruleChunks: [], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), @@ -47,7 +49,7 @@ describe('importRules', () => { }); const ruleChunk = [ruleToImport]; - const result = await importRules({ + const result = await importRulesLegacy({ ruleChunks: [ruleChunk], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), @@ -72,7 +74,7 @@ describe('importRules', () => { }); const ruleChunk = [ruleToImport]; - const result = await importRules({ + const result = await importRulesLegacy({ ruleChunks: [ruleChunk], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), @@ -88,7 +90,7 @@ describe('importRules', () => { immutable: true, version: 1, }; - const result = await importRules({ + const result = await importRulesLegacy({ ruleChunks: [[prebuiltRuleToImport]], overwriteRules: false, detectionRulesClient: context.securitySolution.getDetectionRulesClient(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts similarity index 97% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts index cefe8a0dc4744..7b4c99a5e8fd5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts @@ -18,14 +18,16 @@ import { isRuleConflictError, isRuleImportError } from './errors'; /** * Takes rules to be imported and either creates or updates rules - * based on user overwrite preferences + * based on user overwrite preferences. + * + * @deprecated Use {@link importRules} instead. * @param ruleChunks {@link RuleToImport} - rules being imported * @param overwriteRules {boolean} - whether to overwrite existing rules * with imported rules if their rule_id matches * @param detectionRulesClient {object} * @returns {Promise} an array of error and success messages from import */ -export const importRules = async ({ +export const importRulesLegacy = async ({ ruleChunks, overwriteRules, detectionRulesClient, From b3f1c3dae7a17dbfac9d7567a49f12e2b8be0242 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 17:01:15 -0500 Subject: [PATCH 096/120] Move RuleSourceImporter to logic/import Since this class contains logic for import (which includes prebuilt asset logic), it makes the most sense for it to live here with the other import logic. --- .../api/rules/import_rules/route.ts | 2 +- ...etection_rules_client.import_rules.test.ts | 2 +- .../detection_rules_client_interface.ts | 2 +- .../methods/import_rules.ts | 2 +- .../logic/import/import_rules.test.ts | 2 +- .../logic/import/import_rules.ts | 2 +- .../import}/rule_source_importer/index.ts | 0 .../rule_source_importer.mock.ts | 0 .../rule_source_importer.test.ts | 12 ++++++------ .../rule_source_importer.ts | 19 ++++++++----------- .../import}/rule_source_importer/types.ts | 4 ++-- .../import}/rule_source_importer/utils.ts | 4 ++-- 12 files changed, 24 insertions(+), 27 deletions(-) rename x-pack/plugins/security_solution/server/lib/detection_engine/{prebuilt_rules/logic => rule_management/logic/import}/rule_source_importer/index.ts (100%) rename x-pack/plugins/security_solution/server/lib/detection_engine/{prebuilt_rules/logic => rule_management/logic/import}/rule_source_importer/rule_source_importer.mock.ts (100%) rename x-pack/plugins/security_solution/server/lib/detection_engine/{prebuilt_rules/logic => rule_management/logic/import}/rule_source_importer/rule_source_importer.test.ts (93%) rename x-pack/plugins/security_solution/server/lib/detection_engine/{prebuilt_rules/logic => rule_management/logic/import}/rule_source_importer/rule_source_importer.ts (88%) rename x-pack/plugins/security_solution/server/lib/detection_engine/{prebuilt_rules/logic => rule_management/logic/import}/rule_source_importer/types.ts (83%) rename x-pack/plugins/security_solution/server/lib/detection_engine/{prebuilt_rules/logic => rule_management/logic/import}/rule_source_importer/utils.ts (92%) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index 2bb2224b3e70d..c431f6721caa1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -25,9 +25,9 @@ import { isBulkError, isImportRegular, } from '../../../../routes/utils'; -import { createRuleSourceImporter } from '../../../../prebuilt_rules/logic/rule_source_importer'; import { createPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; +import { createRuleSourceImporter } from '../../../logic/import/rule_source_importer'; import { importRules } from '../../../logic/import/import_rules'; // eslint-disable-next-line no-restricted-imports import { importRulesLegacy } from '../../../logic/import/import_rules_legacy'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts index d09729cf870ba..58b1385dda09c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rules.test.ts @@ -12,7 +12,7 @@ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; import { buildMlAuthz } from '../../../../machine_learning/__mocks__/authz'; import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/mocks'; -import { ruleSourceImporterMock } from '../../../prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock'; +import { ruleSourceImporterMock } from '../import/rule_source_importer/rule_source_importer.mock'; import { createDetectionRulesClient } from './detection_rules_client'; import { importRule } from './methods/import_rule'; import { createRuleImportErrorObject } from '../import/errors'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts index a2f181f607468..53933fa93a4a3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface.ts @@ -14,7 +14,7 @@ import type { RuleToImport, RuleSource, } from '../../../../../../common/api/detection_engine'; -import type { IRuleSourceImporter } from '../../../prebuilt_rules/logic/rule_source_importer'; +import type { IRuleSourceImporter } from '../import/rule_source_importer'; import type { RuleImportErrorObject } from '../import/errors'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts index cf6053b73b510..c5ef14ccf12a8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts @@ -10,7 +10,7 @@ import type { SavedObjectsClientContract } from '@kbn/core/server'; import type { RuleResponse, RuleToImport } from '../../../../../../../common/api/detection_engine'; import { ruleToImportHasVersion } from '../../../../../../../common/api/detection_engine/rule_management'; -import type { IRuleSourceImporter } from '../../../../prebuilt_rules/logic/rule_source_importer'; +import type { IRuleSourceImporter } from '../../import/rule_source_importer'; import { type RuleImportErrorObject, createRuleImportErrorObject, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts index ae32cb7ba8ad2..c6951f6b809f6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts @@ -7,11 +7,11 @@ import { getImportRulesSchemaMock } from '../../../../../../common/api/detection_engine/rule_management/mocks'; import { getRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/rule_response_schema.mock'; -import { ruleSourceImporterMock } from '../../../prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock'; import { importRules } from './import_rules'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { detectionRulesClientMock } from '../detection_rules_client/__mocks__/detection_rules_client'; +import { ruleSourceImporterMock } from './rule_source_importer/rule_source_importer.mock'; import { createRuleImportErrorObject } from './errors'; describe('importRules', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts index 325f1d037a8d8..b7fd7ce3f42fd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts @@ -7,7 +7,7 @@ import type { RuleToImport } from '../../../../../../common/api/detection_engine'; import { type ImportRuleResponse, createBulkErrorObject } from '../../../routes/utils'; -import type { IRuleSourceImporter } from '../../../prebuilt_rules/logic/rule_source_importer'; +import type { IRuleSourceImporter } from './rule_source_importer'; import type { IDetectionRulesClient } from '../detection_rules_client/detection_rules_client_interface'; import { isRuleConflictError, isRuleImportError } from './errors'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/index.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/index.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/index.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.mock.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.mock.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.mock.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts similarity index 93% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts index 7b4c6bffaebea..624f9fd739473 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -5,13 +5,13 @@ * 2.0. */ -import type { ValidatedRuleToImport } from '../../../../../../common/api/detection_engine'; -import { createPrebuiltRuleAssetsClient as createPrebuiltRuleAssetsClientMock } from '../rule_assets/__mocks__/prebuilt_rule_assets_client'; -import type { RuleFromImportStream } from '../../../rule_management/logic/import/utils'; -import { configMock, createMockConfig, requestContextMock } from '../../../routes/__mocks__'; +import type { ValidatedRuleToImport } from '../../../../../../../common/api/detection_engine'; +import { createPrebuiltRuleAssetsClient as createPrebuiltRuleAssetsClientMock } from '../../../../prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client'; +import type { RuleFromImportStream } from '../utils'; +import { configMock, createMockConfig, requestContextMock } from '../../../../routes/__mocks__'; +import { getPrebuiltRuleMock } from '../../../../prebuilt_rules/mocks'; import { createRuleSourceImporter } from './rule_source_importer'; -import * as calculateRuleSourceModule from '../../../rule_management/logic/import/calculate_rule_source_for_import'; -import { getPrebuiltRuleMock } from '../../mocks'; +import * as calculateRuleSourceModule from '../calculate_rule_source_for_import'; describe('ruleSourceImporter', () => { let ruleAssetsClientMock: ReturnType; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts similarity index 88% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts index dda728d673997..5411762da149b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/rule_source_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts @@ -12,20 +12,17 @@ * 2.0. */ -import type { SecuritySolutionApiRequestHandlerContext } from '../../../../../types'; -import type { ConfigType } from '../../../../../config'; +import type { SecuritySolutionApiRequestHandlerContext } from '../../../../../../types'; +import type { ConfigType } from '../../../../../../config'; import type { RuleToImport, ValidatedRuleToImport, -} from '../../../../../../common/api/detection_engine'; -import { calculateRuleSourceForImport } from '../../../rule_management/logic/import/calculate_rule_source_for_import'; -import { - isRuleToImport, - type RuleFromImportStream, -} from '../../../rule_management/logic/import/utils'; -import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; -import type { IPrebuiltRuleAssetsClient } from '../rule_assets/prebuilt_rule_assets_client'; -import { ensureLatestRulesPackageInstalled } from '../ensure_latest_rules_package_installed'; +} from '../../../../../../../common/api/detection_engine'; +import type { PrebuiltRuleAsset } from '../../../../prebuilt_rules'; +import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; +import { ensureLatestRulesPackageInstalled } from '../../../../prebuilt_rules/logic/ensure_latest_rules_package_installed'; +import { calculateRuleSourceForImport } from '../calculate_rule_source_for_import'; +import { isRuleToImport, type RuleFromImportStream } from '../utils'; import type { CalculatedRuleSource, IRuleSourceImporter, RuleSpecifier } from './types'; import { fetchAvailableRuleAssetIds, fetchMatchingAssets } from './utils'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/types.ts similarity index 83% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/types.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/types.ts index f588b36ca2b74..d1dc32b4129bd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/types.ts @@ -8,8 +8,8 @@ import type { RuleSource, ValidatedRuleToImport, -} from '../../../../../../common/api/detection_engine'; -import type { RuleFromImportStream } from '../../../rule_management/logic/import/utils'; +} from '../../../../../../../common/api/detection_engine'; +import type { RuleFromImportStream } from '../utils'; export interface RuleSpecifier { rule_id: string; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts similarity index 92% rename from x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts index da59a2d1e649c..a9e077202b2fc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_source_importer/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts @@ -5,8 +5,8 @@ * 2.0. */ -import type { PrebuiltRuleAsset } from '../../model/rule_assets/prebuilt_rule_asset'; -import type { IPrebuiltRuleAssetsClient } from '../rule_assets/prebuilt_rule_assets_client'; +import type { PrebuiltRuleAsset } from '../../../../prebuilt_rules'; +import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import type { RuleSpecifier } from './types'; /** From 82355a500ed1ac6d8710d9311cee7fd8b8215687 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 17:33:27 -0500 Subject: [PATCH 097/120] Simplify RuleSourceImporter interface/logic With the refactor of the import route to handle parser errors before passing successful rules to the import function(s), the Importer no longer needs to handle RuleFromImportStream. This also cleans up a few unnecessary complexities: * Don't make enablement a responsibility of the client * Don't swallow errors during setup() * Allow setup() to be called with an empty array --- .../rule_source_importer.test.ts | 32 ++++++++--------- .../rule_source_importer.ts | 35 +++---------------- .../import/rule_source_importer/types.ts | 6 ++-- 3 files changed, 24 insertions(+), 49 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts index 624f9fd739473..e593e5c8481bd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -5,9 +5,11 @@ * 2.0. */ -import type { ValidatedRuleToImport } from '../../../../../../../common/api/detection_engine'; +import type { + RuleToImport, + ValidatedRuleToImport, +} from '../../../../../../../common/api/detection_engine'; import { createPrebuiltRuleAssetsClient as createPrebuiltRuleAssetsClientMock } from '../../../../prebuilt_rules/logic/rule_assets/__mocks__/prebuilt_rule_assets_client'; -import type { RuleFromImportStream } from '../utils'; import { configMock, createMockConfig, requestContextMock } from '../../../../routes/__mocks__'; import { getPrebuiltRuleMock } from '../../../../prebuilt_rules/mocks'; import { createRuleSourceImporter } from './rule_source_importer'; @@ -17,7 +19,7 @@ describe('ruleSourceImporter', () => { let ruleAssetsClientMock: ReturnType; let config: ReturnType; let context: ReturnType['securitySolution']; - let ruleToImport: RuleFromImportStream; + let ruleToImport: RuleToImport; let subject: ReturnType; beforeEach(() => { @@ -27,7 +29,8 @@ describe('ruleSourceImporter', () => { context = requestContextMock.create().securitySolution; ruleAssetsClientMock = createPrebuiltRuleAssetsClientMock(); ruleAssetsClientMock.fetchLatestAssets.mockResolvedValue([{}]); - ruleToImport = { rule_id: 'rule-1', version: 1 } as RuleFromImportStream; + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([]); + ruleToImport = { rule_id: 'rule-1', version: 1 } as RuleToImport; subject = createRuleSourceImporter({ context, @@ -45,17 +48,6 @@ describe('ruleSourceImporter', () => { }); describe('#setup()', () => { - it('does nothing if feature flag is disabled', async () => { - const disabledSubject = createRuleSourceImporter({ - config: createMockConfig(), - context, - prebuiltRuleAssetsClient: ruleAssetsClientMock, - }); - await disabledSubject.setup({ rules: [ruleToImport] }); - - expect(ruleAssetsClientMock.fetchLatestAssets).toHaveBeenCalledTimes(0); - }); - it('fetches the rules package on the initial call', async () => { await subject.setup({ rules: [] }); @@ -69,6 +61,14 @@ describe('ruleSourceImporter', () => { expect(ruleAssetsClientMock.fetchLatestAssets).toHaveBeenCalledTimes(1); }); + + it('throws an error if the ruleAsstClient does', () => { + ruleAssetsClientMock.fetchLatestAssets.mockReset().mockRejectedValue(new Error('failed')); + + expect(() => subject.setup({ rules: [] })).rejects.toThrowErrorMatchingInlineSnapshot( + `"failed"` + ); + }); }); describe('#isPrebuiltRule()', () => { @@ -93,7 +93,7 @@ describe('ruleSourceImporter', () => { await subject.setup({ rules: [ruleToImport] }); expect(() => - subject.isPrebuiltRule({ rule_id: 'other-rule' } as RuleFromImportStream) + subject.isPrebuiltRule({ rule_id: 'other-rule' } as RuleToImport) ).toThrowErrorMatchingInlineSnapshot(`"Rule other-rule was not registered during setup."`); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts index 5411762da149b..c03b7ca60ac3d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts @@ -22,7 +22,6 @@ import type { PrebuiltRuleAsset } from '../../../../prebuilt_rules'; import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import { ensureLatestRulesPackageInstalled } from '../../../../prebuilt_rules/logic/ensure_latest_rules_package_installed'; import { calculateRuleSourceForImport } from '../calculate_rule_source_for_import'; -import { isRuleToImport, type RuleFromImportStream } from '../utils'; import type { CalculatedRuleSource, IRuleSourceImporter, RuleSpecifier } from './types'; import { fetchAvailableRuleAssetIds, fetchMatchingAssets } from './utils'; @@ -37,7 +36,6 @@ export class RuleSourceImporter implements IRuleSourceImporter { private context: SecuritySolutionApiRequestHandlerContext; private config: ConfigType; private ruleAssetsClient: IPrebuiltRuleAssetsClient; - private enabled: boolean; private latestPackagesInstalled: boolean = false; private matchingAssets: PrebuiltRuleAsset[] = []; private knownRules: RuleSpecifier[] = []; @@ -55,7 +53,6 @@ export class RuleSourceImporter implements IRuleSourceImporter { this.ruleAssetsClient = prebuiltRuleAssetsClient; this.context = context; this.config = config; - this.enabled = config.experimentalFeatures.prebuiltRulesCustomizationEnabled; } /** @@ -63,34 +60,18 @@ export class RuleSourceImporter implements IRuleSourceImporter { * Prepares the importing of rules by ensuring the latest rules * package is installed and fetching the associated prebuilt rule assets. */ - public async setup({ rules }: { rules: RuleFromImportStream[] }): Promise { - if (!this.enabled) { - return; - } - + public async setup({ rules }: { rules: RuleToImport[] }): Promise { if (!this.latestPackagesInstalled) { await ensureLatestRulesPackageInstalled(this.ruleAssetsClient, this.config, this.context); this.latestPackagesInstalled = true; } - this.knownRules = rules - .filter(isRuleToImport) - .map((rule) => ({ rule_id: rule.rule_id, version: rule.version })); - - try { - this.validateSetupState(); - - this.matchingAssets = await this.fetchMatchingAssets(); - this.availableRuleAssetIds = await this.fetchAvailableRuleAssetIds(); - } catch (e) { - // If all incoming rules were errors, there's nothing to calculate - } + this.knownRules = rules.map((rule) => ({ rule_id: rule.rule_id, version: rule.version })); + this.matchingAssets = await this.fetchMatchingAssets(); + this.availableRuleAssetIds = await this.fetchAvailableRuleAssetIds(); } - public isPrebuiltRule(rule: RuleFromImportStream): boolean { - if (!isRuleToImport(rule)) { - return false; - } + public isPrebuiltRule(rule: RuleToImport): boolean { this.validateRuleInput(rule); return this.availableRuleAssetIds.includes(rule.rule_id); @@ -131,12 +112,6 @@ export class RuleSourceImporter implements IRuleSourceImporter { if (!this.latestPackagesInstalled) { throw new Error('Expected rules package to be installed'); } - - if (this.knownRules.length === 0) { - throw new Error( - 'Expected rules to have been registered, but none were. Please call the setup() method with the relevant incoming rules.' - ); - } } private validateRuleInput(rule: RuleToImport) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/types.ts index d1dc32b4129bd..de5e52956ac16 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/types.ts @@ -7,9 +7,9 @@ import type { RuleSource, + RuleToImport, ValidatedRuleToImport, } from '../../../../../../../common/api/detection_engine'; -import type { RuleFromImportStream } from '../utils'; export interface RuleSpecifier { rule_id: string; @@ -22,7 +22,7 @@ export interface CalculatedRuleSource { } export interface IRuleSourceImporter { - setup: ({ rules }: { rules: RuleFromImportStream[] }) => Promise; - isPrebuiltRule: (rule: RuleFromImportStream) => boolean; + setup: ({ rules }: { rules: RuleToImport[] }) => Promise; + isPrebuiltRule: (rule: RuleToImport) => boolean; calculateRuleSource: (rule: ValidatedRuleToImport) => CalculatedRuleSource; } From 30bf977bf26697cba5996e9221e83d06867aa20d Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 18:01:47 -0500 Subject: [PATCH 098/120] Prevent unnecessary looping during import calculations This simplifies the calculation of rule_source in cases where an asset is found matching the `rule_id` but not the `version`. The logic is the same, but we now use a Set() of `rule_id`s and perform a O(1) check with `Set.has()` instead of performing `Array.includes` (O(n)) for each rule. Since `calculateRuleSourceForImport` was previously performing this same logic, and the Importer is the only caller, it's been updated to instead receive a boolean representing the result of that calculation. --- .../calculate_rule_source_for_import.test.ts | 8 ++++---- .../import/calculate_rule_source_for_import.ts | 8 ++++---- .../rule_source_importer.test.ts | 17 +++++++---------- .../rule_source_importer.ts | 8 ++++---- 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts index c982a9b790b3b..d22c31295db4e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -14,7 +14,7 @@ describe('calculateRuleSourceForImport', () => { const result = calculateRuleSourceForImport({ rule: getRulesSchemaMock(), prebuiltRuleAssets: [], - installedRuleIds: [], + ruleIdExists: false, }); expect(result).toEqual({ @@ -32,7 +32,7 @@ describe('calculateRuleSourceForImport', () => { const result = calculateRuleSourceForImport({ rule, prebuiltRuleAssets: [], - installedRuleIds: ['rule_id'], + ruleIdExists: true, }); expect(result).toEqual({ @@ -52,7 +52,7 @@ describe('calculateRuleSourceForImport', () => { const result = calculateRuleSourceForImport({ rule, prebuiltRuleAssets, - installedRuleIds: ['rule_id'], + ruleIdExists: true, }); expect(result).toEqual({ @@ -72,7 +72,7 @@ describe('calculateRuleSourceForImport', () => { const result = calculateRuleSourceForImport({ rule, prebuiltRuleAssets, - installedRuleIds: ['rule_id'], + ruleIdExists: true, }); expect(result).toEqual({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 38669e67a9665..4d877708fe6f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -18,23 +18,23 @@ import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset * @param rule The rule to be imported * @param prebuiltRuleAssets A list of prebuilt rule assets, which may include * the installed version of the specified prebuilt rule. - * @param installedRuleIds A list of prebuilt rule IDs that are currently installed + * @param ruleIdExists {boolean} Whether the rule's rule_id is available as a + * prebuilt asset (independent of the specified version). * * @returns The calculated rule_source and immutable fields for the rule */ export const calculateRuleSourceForImport = ({ rule, prebuiltRuleAssets, - installedRuleIds, + ruleIdExists, }: { rule: ValidatedRuleToImport; prebuiltRuleAssets: PrebuiltRuleAsset[]; - installedRuleIds: string[]; + ruleIdExists: boolean; }): { ruleSource: RuleSource; immutable: boolean } => { const assetWithMatchingVersion = prebuiltRuleAssets.find( (asset) => asset.rule_id === rule.rule_id ); - const ruleIdExists = installedRuleIds.includes(rule.rule_id); const ruleSource = calculateRuleSourceFromAsset({ rule, assetWithMatchingVersion, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts index e593e5c8481bd..c1898f97a40ad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -106,6 +106,7 @@ describe('ruleSourceImporter', () => { describe('#calculateRuleSource()', () => { let rule: ValidatedRuleToImport; + let calculatorSpy: jest.SpyInstance; beforeEach(() => { rule = { rule_id: 'validated-rule', version: 1 } as ValidatedRuleToImport; @@ -115,14 +116,14 @@ describe('ruleSourceImporter', () => { ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ getPrebuiltRuleMock({ rule_id: 'rule-1' }), getPrebuiltRuleMock({ rule_id: 'rule-2' }), + getPrebuiltRuleMock({ rule_id: 'validated-rule' }), ]); - }); - - it('invokes calculateRuleSourceForImport with the correct arguments', async () => { - const calculatorSpy = jest + calculatorSpy = jest .spyOn(calculateRuleSourceModule, 'calculateRuleSourceForImport') .mockReturnValue({ ruleSource: { type: 'internal' }, immutable: false }); + }); + it('invokes calculateRuleSourceForImport with the correct arguments', async () => { await subject.setup({ rules: [rule] }); await subject.calculateRuleSource(rule); @@ -130,7 +131,7 @@ describe('ruleSourceImporter', () => { expect(calculatorSpy).toHaveBeenCalledWith({ rule, prebuiltRuleAssets: [expect.objectContaining({ rule_id: 'rule-1' })], - installedRuleIds: ['rule-1', 'rule-2'], + ruleIdExists: true, }); }); @@ -151,10 +152,6 @@ describe('ruleSourceImporter', () => { describe('for rules set up without a version', () => { it('invokes the calculator with the correct arguments', async () => { - const calculatorSpy = jest - .spyOn(calculateRuleSourceModule, 'calculateRuleSourceForImport') - .mockReturnValue({ ruleSource: { type: 'internal' }, immutable: false }); - await subject.setup({ rules: [{ ...rule, version: undefined }] }); await subject.calculateRuleSource(rule); @@ -162,7 +159,7 @@ describe('ruleSourceImporter', () => { expect(calculatorSpy).toHaveBeenCalledWith({ rule, prebuiltRuleAssets: [expect.objectContaining({ rule_id: 'rule-1' })], - installedRuleIds: ['rule-1', 'rule-2'], + ruleIdExists: true, }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts index c03b7ca60ac3d..522c67a64f717 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts @@ -39,7 +39,7 @@ export class RuleSourceImporter implements IRuleSourceImporter { private latestPackagesInstalled: boolean = false; private matchingAssets: PrebuiltRuleAsset[] = []; private knownRules: RuleSpecifier[] = []; - private availableRuleAssetIds: string[] = []; + private availableRuleAssetIds: Set = new Set(); constructor({ config, @@ -68,13 +68,13 @@ export class RuleSourceImporter implements IRuleSourceImporter { this.knownRules = rules.map((rule) => ({ rule_id: rule.rule_id, version: rule.version })); this.matchingAssets = await this.fetchMatchingAssets(); - this.availableRuleAssetIds = await this.fetchAvailableRuleAssetIds(); + this.availableRuleAssetIds = new Set(await this.fetchAvailableRuleAssetIds()); } public isPrebuiltRule(rule: RuleToImport): boolean { this.validateRuleInput(rule); - return this.availableRuleAssetIds.includes(rule.rule_id); + return this.availableRuleAssetIds.has(rule.rule_id); } public calculateRuleSource(rule: ValidatedRuleToImport): CalculatedRuleSource { @@ -83,7 +83,7 @@ export class RuleSourceImporter implements IRuleSourceImporter { return calculateRuleSourceForImport({ rule, prebuiltRuleAssets: this.matchingAssets, - installedRuleIds: this.availableRuleAssetIds, + ruleIdExists: this.availableRuleAssetIds.has(rule.rule_id), }); } From 6f7d10c6d82f969b50e1d0c7a868e7e59d6a7d1e Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 18:07:10 -0500 Subject: [PATCH 099/120] Revert changes to our RuleResponse ordering This change is no longer needed since our `internalRuleToApiResponse` method is no longer changed. --- .../cypress/objects/rule.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts index 86558c3892c98..50e358515d922 100644 --- a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts @@ -576,26 +576,26 @@ export const expectedExportedRule = (ruleResponse: Cypress.Response Date: Fri, 27 Sep 2024 20:32:49 -0500 Subject: [PATCH 100/120] Update test descriptions for clarity In fact, this clarified a duplicate test (which was removed in this commit). --- .../import_rules/rule_to_import.test.ts | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts index e3bd7992398a4..e945683fc5019 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts @@ -550,7 +550,7 @@ describe('RuleToImport', () => { ); }); - test('You cannot set the immutable to a number when trying to create a rule', () => { + test('You cannot set immutable to a number', () => { const payload = getImportRulesSchemaMock({ // @ts-expect-error assign unsupported value immutable: 5, @@ -564,7 +564,7 @@ describe('RuleToImport', () => { ); }); - test('You can optionally set the immutable to be false', () => { + test('You can optionally set immutable to false', () => { const payload: RuleToImportInput = getImportRulesSchemaMock({ immutable: false, }); @@ -574,7 +574,7 @@ describe('RuleToImport', () => { expectParseSuccess(result); }); - test('You can optionally set the immutable to be true', () => { + test('You can optionally set immutable to true', () => { const payload = getImportRulesSchemaMock({ immutable: true, }); @@ -584,20 +584,6 @@ describe('RuleToImport', () => { expectParseSuccess(result); }); - test('You cannot set the immutable to be a number', () => { - const payload = getImportRulesSchemaMock({ - // @ts-expect-error assign unsupported value - immutable: 5, - }); - - const result = RuleToImport.safeParse(payload); - expectParseError(result); - - expect(stringifyZodError(result.error)).toMatchInlineSnapshot( - `"immutable: Expected boolean, received number"` - ); - }); - test('You cannot set the risk_score to 101', () => { const payload: RuleToImportInput = getImportRulesSchemaMock({ risk_score: 101, From cd4bc548b5e300d5e38d6bdcaf73ddaa9aaadf16 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 20:39:22 -0500 Subject: [PATCH 101/120] style: More accurate variable name I'd previously renamed this to `importedRules` from `importRules`, but they're both innacurate. Instead, just calling it what it is. --- .../rule_management/api/rules/import_rules/route.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index c431f6721caa1..e9131050d9629 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -201,7 +201,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C } }); - const importedRules: ImportRulesResponse = { + const importRulesResponse: ImportRulesResponse = { success: errors.length === 0, success_count: successes.length, rules_count: rules.length, @@ -215,7 +215,7 @@ export const importRulesRoute = (router: SecuritySolutionPluginRouter, config: C action_connectors_warnings: actionConnectorWarnings, }; - return response.ok({ body: ImportRulesResponse.parse(importedRules) }); + return response.ok({ body: ImportRulesResponse.parse(importRulesResponse) }); } catch (err) { const error = transformError(err); return siemResponse.error({ From 3cdbd3bff83ab8afc2e23d74ff707cfbfc6dbd8d Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 21:04:16 -0500 Subject: [PATCH 102/120] Fix outstanding TODO/skipped test This test was failing due to the stream not containing a terminating character (null). Fixing it also revealed an interesting behavior (and potentially a bug) around the way we validate the object limit. --- ...te_promise_from_rule_import_stream.test.ts | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts index 8e0bacbd5206a..cc5ffc7d48bd9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_promise_from_rule_import_stream.test.ts @@ -81,20 +81,39 @@ describe('createPromiseFromRuleImportStream', () => { ]); }); - // TODO - Yara - there's a integration test testing this, but causing timeouts here - test.skip('returns error when ndjson stream is larger than limit', async () => { + test('throws an error when the number of rules in the stream is equal to the limit', async () => { const sample1 = getOutputSample(); const sample2 = getOutputSample(); sample2.rule_id = 'rule-2'; const ndJsonStream = new Readable({ read() { this.push(getSampleAsNdjson(sample1)); + this.push('\n'); this.push(getSampleAsNdjson(sample2)); + this.push(null); }, }); await expect( createPromiseFromRuleImportStream({ stream: ndJsonStream, objectLimit: 2 }) + ).rejects.toThrowError("Can't import more than 2 rules"); + }); + + test('throws an error when the number of rules in the stream is larger than the limit', async () => { + const sample1 = getOutputSample(); + const sample2 = getOutputSample(); + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push('\n'); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + + await expect( + createPromiseFromRuleImportStream({ stream: ndJsonStream, objectLimit: 1 }) ).rejects.toThrowError("Can't import more than 1 rules"); }); From d06a1e368490ad2b9e3aba02ec88e13c9f63380b Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 21:16:59 -0500 Subject: [PATCH 103/120] Test cleanup Moving shared test setup (representing the happy path) in the (previously empty?) beforeEach. --- .../rule_source_importer/rule_source_importer.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts index c1898f97a40ad..e001a8936248e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -72,24 +72,24 @@ describe('ruleSourceImporter', () => { }); describe('#isPrebuiltRule()', () => { - beforeEach(() => {}); + beforeEach(() => { + ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ruleToImport]); + }); it("returns false if the rule's rule_id doesn't match an available rule asset", async () => { - ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([]); + ruleAssetsClientMock.fetchLatestVersions.mockReset().mockResolvedValue([]); await subject.setup({ rules: [ruleToImport] }); expect(subject.isPrebuiltRule(ruleToImport)).toBe(false); }); it("returns true if the rule's rule_id matches an available rule asset", async () => { - ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ruleToImport]); await subject.setup({ rules: [ruleToImport] }); expect(subject.isPrebuiltRule(ruleToImport)).toBe(true); }); it('throws an error if the rule is not known to the calculator', async () => { - ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ruleToImport]); await subject.setup({ rules: [ruleToImport] }); expect(() => From c8ab8e802a6c47ebc728b171166863c4b8d05080 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 27 Sep 2024 22:29:02 -0500 Subject: [PATCH 104/120] Fix linter warning about floating promise --- .../import/rule_source_importer/rule_source_importer.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts index e001a8936248e..9c41ea77bfafd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -62,10 +62,10 @@ describe('ruleSourceImporter', () => { expect(ruleAssetsClientMock.fetchLatestAssets).toHaveBeenCalledTimes(1); }); - it('throws an error if the ruleAsstClient does', () => { + it('throws an error if the ruleAsstClient does', async () => { ruleAssetsClientMock.fetchLatestAssets.mockReset().mockRejectedValue(new Error('failed')); - expect(() => subject.setup({ rules: [] })).rejects.toThrowErrorMatchingInlineSnapshot( + await expect(() => subject.setup({ rules: [] })).rejects.toThrowErrorMatchingInlineSnapshot( `"failed"` ); }); From 62c3bf3d3fe908014b220b24e260f991d5d2c732 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 4 Oct 2024 12:23:27 -0500 Subject: [PATCH 105/120] Remove outdated comment Version was required in a previous implementation on this branch, but we've reverted those changes. --- .../rule_management/import_rules/rule_to_import.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts index 0d69848b50175..3372d04d06652 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.ts @@ -18,7 +18,6 @@ import { /** * Differences from this and the createRulesSchema are * - rule_id is required - * - version is required * - id is optional (but ignored in the import code - rule_id is exclusively used for imports) * - immutable is optional (but ignored in the import code) * - created_at is optional (but ignored in the import code) From f9b14068b49c2994fc9034e84bccb3e2d9e9b052 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 4 Oct 2024 12:50:31 -0500 Subject: [PATCH 106/120] Revert "Define DiffableRuleInput as type used for calculating rule_source" This reverts commit 538620cf4132157fc1dcdde5bdc50c6904efce90. We opted to continue standardizing on RuleResponse for our diffing/rule_source calculations, since that is the standard, superset representation of a rule. Conflicts: x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts --- .../diff/convert_rule_to_diffable.ts | 6 +++--- .../diff/extract_building_block_object.ts | 6 ++---- .../diff/extract_rule_name_override_object.ts | 4 ++-- .../prebuilt_rules/diff/extract_rule_schedule.ts | 5 ++--- .../diff/extract_timeline_template_reference.ts | 4 ++-- .../diff/extract_timestamp_override_object.ts | 4 ++-- .../detection_engine/prebuilt_rules/diff/types.ts | 14 -------------- .../mergers/rule_source/calculate_is_customized.ts | 4 ++-- .../import/calculate_rule_source_for_import.ts | 8 +++++++- .../import/calculate_rule_source_from_asset.ts | 5 ++--- 10 files changed, 24 insertions(+), 36 deletions(-) delete mode 100644 x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/types.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts index 77b8011e87b87..45b4612e83c8e 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts @@ -20,6 +20,7 @@ import type { NewTermsRuleCreateProps, QueryRule, QueryRuleCreateProps, + RuleResponse, SavedQueryRule, SavedQueryRuleCreateProps, ThreatMatchRule, @@ -40,7 +41,6 @@ import type { DiffableThresholdFields, } from '../../../api/detection_engine/prebuilt_rules'; import { addEcsToRequiredFields } from '../../rule_management/utils'; -import type { DiffableRuleInput } from './types'; import { extractBuildingBlockObject } from './extract_building_block_object'; import { extractInlineKqlQuery, @@ -58,7 +58,7 @@ import { extractTimestampOverrideObject } from './extract_timestamp_override_obj * Normalizes a given rule to the form which is suitable for passing to the diff algorithm. * Read more in the JSDoc description of DiffableRule. */ -export const convertRuleToDiffable = (rule: DiffableRuleInput): DiffableRule => { +export const convertRuleToDiffable = (rule: RuleResponse): DiffableRule => { const commonFields = extractDiffableCommonFields(rule); switch (rule.type) { @@ -108,7 +108,7 @@ export const convertRuleToDiffable = (rule: DiffableRuleInput): DiffableRule => }; const extractDiffableCommonFields = ( - rule: DiffableRuleInput + rule: RuleResponse ): RequiredOptional => { return { // --------------------- REQUIRED FIELDS diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_building_block_object.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_building_block_object.ts index eae1350471701..18d6ebfb54ce4 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_building_block_object.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_building_block_object.ts @@ -5,12 +5,10 @@ * 2.0. */ +import type { RuleResponse } from '../../../api/detection_engine/model/rule_schema'; import type { BuildingBlockObject } from '../../../api/detection_engine/prebuilt_rules'; -import type { DiffableRuleInput } from './types'; -export const extractBuildingBlockObject = ( - rule: DiffableRuleInput -): BuildingBlockObject | undefined => { +export const extractBuildingBlockObject = (rule: RuleResponse): BuildingBlockObject | undefined => { if (rule.building_block_type == null) { return undefined; } diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_name_override_object.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_name_override_object.ts index 13eb32f4c794f..55119608264f9 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_name_override_object.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_name_override_object.ts @@ -5,11 +5,11 @@ * 2.0. */ +import type { RuleResponse } from '../../../api/detection_engine/model/rule_schema'; import type { RuleNameOverrideObject } from '../../../api/detection_engine/prebuilt_rules'; -import type { DiffableRuleInput } from './types'; export const extractRuleNameOverrideObject = ( - rule: DiffableRuleInput + rule: RuleResponse ): RuleNameOverrideObject | undefined => { if (rule.rule_name_override == null) { return undefined; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_schedule.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_schedule.ts index 5ebf1e96c0769..6812a0a2f6fc6 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_schedule.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_schedule.ts @@ -9,11 +9,10 @@ import moment from 'moment'; import dateMath from '@elastic/datemath'; import { parseDuration } from '@kbn/alerting-plugin/common'; -import type { RuleMetadata } from '../../../api/detection_engine/model/rule_schema'; +import type { RuleMetadata, RuleResponse } from '../../../api/detection_engine/model/rule_schema'; import type { RuleSchedule } from '../../../api/detection_engine/prebuilt_rules'; -import type { DiffableRuleInput } from './types'; -export const extractRuleSchedule = (rule: DiffableRuleInput): RuleSchedule => { +export const extractRuleSchedule = (rule: RuleResponse): RuleSchedule => { const interval = rule.interval ?? '5m'; const from = rule.from ?? 'now-6m'; const to = rule.to ?? 'now'; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timeline_template_reference.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timeline_template_reference.ts index d347144dd4da3..7dcce30061659 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timeline_template_reference.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timeline_template_reference.ts @@ -5,11 +5,11 @@ * 2.0. */ +import type { RuleResponse } from '../../../api/detection_engine/model/rule_schema'; import type { TimelineTemplateReference } from '../../../api/detection_engine/prebuilt_rules'; -import type { DiffableRuleInput } from './types'; export const extractTimelineTemplateReference = ( - rule: DiffableRuleInput + rule: RuleResponse ): TimelineTemplateReference | undefined => { if (rule.timeline_id == null) { return undefined; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timestamp_override_object.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timestamp_override_object.ts index 588df49b9b969..40f218fe430b7 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timestamp_override_object.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_timestamp_override_object.ts @@ -5,11 +5,11 @@ * 2.0. */ +import type { RuleResponse } from '../../../api/detection_engine/model/rule_schema'; import type { TimestampOverrideObject } from '../../../api/detection_engine/prebuilt_rules'; -import type { DiffableRuleInput } from './types'; export const extractTimestampOverrideObject = ( - rule: DiffableRuleInput + rule: RuleResponse ): TimestampOverrideObject | undefined => { if (rule.timestamp_override == null) { return undefined; diff --git a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/types.ts b/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/types.ts deleted file mode 100644 index 46ed266a941be..0000000000000 --- a/x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/types.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { ValidatedRuleToImport } from '../../../api/detection_engine'; - -/** - * This type represents the minimal rule representation that can be - * converted to the DiffableRule type. - */ -export type DiffableRuleInput = ValidatedRuleToImport; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts index 88b47063e78df..92c7d941dd7f1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { DiffableRuleInput } from '../../../../../../../../common/detection_engine/prebuilt_rules/diff/types'; +import type { RuleResponse } from '../../../../../../../../common/api/detection_engine'; import { MissingVersion } from '../../../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import { calculateRuleFieldsDiff } from '../../../../../prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff'; @@ -14,7 +14,7 @@ import { convertPrebuiltRuleAssetToRuleResponse } from '../../converters/convert export function calculateIsCustomized( baseRule: PrebuiltRuleAsset | undefined, - nextRule: DiffableRuleInput + nextRule: RuleResponse ) { if (baseRule == null) { // If the base version is missing, we consider the rule to be customized diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 4d877708fe6f8..43922b99c1c69 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -10,6 +10,7 @@ import type { ValidatedRuleToImport, } from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; +import { convertPrebuiltRuleAssetToRuleResponse } from '../detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response'; import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset'; /** @@ -35,8 +36,13 @@ export const calculateRuleSourceForImport = ({ const assetWithMatchingVersion = prebuiltRuleAssets.find( (asset) => asset.rule_id === rule.rule_id ); + // We convert here so that RuleSource calculation can + // continue to deal only with RuleResponses. The fields missing from the + // incoming rule are not actually needed for the calculation, but only to + // satisfy the type system. + const ruleResponseForImport = convertPrebuiltRuleAssetToRuleResponse(rule); const ruleSource = calculateRuleSourceFromAsset({ - rule, + rule: ruleResponseForImport, assetWithMatchingVersion, ruleIdExists, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts index 0b45139e5aa08..09b284de60ddc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts @@ -5,8 +5,7 @@ * 2.0. */ -import type { RuleSource } from '../../../../../../common/api/detection_engine'; -import type { DiffableRuleInput } from '../../../../../../common/detection_engine/prebuilt_rules/diff/types'; +import type { RuleResponse, RuleSource } from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; import { calculateIsCustomized } from '../detection_rules_client/mergers/rule_source/calculate_is_customized'; @@ -26,7 +25,7 @@ export const calculateRuleSourceFromAsset = ({ assetWithMatchingVersion, ruleIdExists, }: { - rule: DiffableRuleInput; + rule: RuleResponse; assetWithMatchingVersion: PrebuiltRuleAsset | undefined; ruleIdExists: boolean; }): RuleSource => { From 0878638d30d7d00fae9f26bd76539b799ed88980 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 8 Oct 2024 16:17:31 -0500 Subject: [PATCH 107/120] style: replace mutative while loop with for..of loop --- .../rule_management/logic/import/import_rules.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts index b7fd7ce3f42fd..c77c50e6d6ca1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts @@ -41,9 +41,7 @@ export const importRules = async ({ return response; } - while (ruleChunks.length) { - const rules = ruleChunks.shift() ?? []; - + for (const rules of ruleChunks) { const importedRulesResponse = await detectionRulesClient.importRules({ allowMissingConnectorSecrets, overwriteRules, From 6fdcf4af0945b0c48f8d57407132ac6589c2e140 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 8 Oct 2024 16:21:09 -0500 Subject: [PATCH 108/120] Adds explicit return type as a best practice Lets someone accidentally change the interface later on. --- .../rule_management/logic/import/import_rules.ts | 2 +- .../rule_management/logic/import/import_rules_legacy.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts index c77c50e6d6ca1..385b102ba443f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts @@ -32,7 +32,7 @@ export const importRules = async ({ detectionRulesClient: IDetectionRulesClient; ruleSourceImporter: IRuleSourceImporter; allowMissingConnectorSecrets?: boolean; -}) => { +}): Promise => { const response: ImportRuleResponse[] = []; // If we had 100% errors and no successful rule could be imported we still have to output an error. diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts index 7b4c99a5e8fd5..f790cd98a3fb3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts @@ -39,7 +39,7 @@ export const importRulesLegacy = async ({ detectionRulesClient: IDetectionRulesClient; allowMissingConnectorSecrets?: boolean; savedObjectsClient: SavedObjectsClientContract; -}) => { +}): Promise => { const response: ImportRuleResponse[] = []; // If we had 100% errors and no successful rule could be imported we still have to output an error. From 0d876c19c86e9d3f1b79e8b88eb4112810df2f8d Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 8 Oct 2024 16:25:38 -0500 Subject: [PATCH 109/120] Remove unhelpful comment(s) Both the new and legacy code contain this comment (which was introduced in https://github.com/elastic/kibana/pull/118816/files#diff-25733abc36ad7c5867db625af8b042884f967bfed4fa50b173cde94c8fbaf87cR66), but doesn't add any useful information beyond the general "base case" return clause that it clearly is. --- .../rule_management/logic/import/import_rules.ts | 2 -- .../rule_management/logic/import/import_rules_legacy.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts index 385b102ba443f..012c64c0ad8a4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.ts @@ -35,8 +35,6 @@ export const importRules = async ({ }): Promise => { const response: ImportRuleResponse[] = []; - // If we had 100% errors and no successful rule could be imported we still have to output an error. - // otherwise we would output we are success importing 0 rules. if (ruleChunks.length === 0) { return response; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts index f790cd98a3fb3..384683ce1916e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_legacy.ts @@ -42,8 +42,6 @@ export const importRulesLegacy = async ({ }): Promise => { const response: ImportRuleResponse[] = []; - // If we had 100% errors and no successful rule could be imported we still have to output an error. - // otherwise we would output we are success importing 0 rules. if (ruleChunks.length === 0) { return response; } From 6f4f770bf9a1a9bbf56a2ab84bb357cf5dcb9cdd Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 8 Oct 2024 16:44:28 -0500 Subject: [PATCH 110/120] Add test demonstrating that `version` is not required For determination of a prebuilt rule. --- .../rule_source_importer/rule_source_importer.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts index 9c41ea77bfafd..78cc2b0a2e279 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -89,6 +89,13 @@ describe('ruleSourceImporter', () => { expect(subject.isPrebuiltRule(ruleToImport)).toBe(true); }); + it('returns true if the rule has no version, but its rule_id matches an available rule asset', async () => { + const ruleWithoutVersion = { ...ruleToImport, version: undefined }; + await subject.setup({ rules: [ruleWithoutVersion] }); + + expect(subject.isPrebuiltRule(ruleWithoutVersion)).toBe(true); + }); + it('throws an error if the rule is not known to the calculator', async () => { await subject.setup({ rules: [ruleToImport] }); From f17fb849cd5b363f817c433da9ab6839190ffc70 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 8 Oct 2024 17:07:47 -0500 Subject: [PATCH 111/120] Save some unnecessary looping through prebuilt assets Our Importer now makes a single pass over the assets, and stores them in a map by `rule_id`, so that `calculateRuleSourceForImport` can simply pluck the appropriate asset out for calculation. --- .../calculate_rule_source_for_import.test.ts | 12 ++++++------ .../import/calculate_rule_source_for_import.ts | 8 +++----- .../rule_source_importer.test.ts | 5 +++-- .../rule_source_importer/rule_source_importer.ts | 16 ++++++++++------ 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts index d22c31295db4e..27999dda27a4f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -13,7 +13,7 @@ describe('calculateRuleSourceForImport', () => { it('calculates as internal if no asset is found', () => { const result = calculateRuleSourceForImport({ rule: getRulesSchemaMock(), - prebuiltRuleAssets: [], + prebuiltRuleAssetsByRuleId: {}, ruleIdExists: false, }); @@ -31,7 +31,7 @@ describe('calculateRuleSourceForImport', () => { const result = calculateRuleSourceForImport({ rule, - prebuiltRuleAssets: [], + prebuiltRuleAssetsByRuleId: {}, ruleIdExists: true, }); @@ -47,11 +47,11 @@ describe('calculateRuleSourceForImport', () => { it('calculates as external with customizations if a matching asset/version is found', () => { const rule = getRulesSchemaMock(); rule.rule_id = 'rule_id'; - const prebuiltRuleAssets = [getPrebuiltRuleMock({ rule_id: 'rule_id' })]; + const prebuiltRuleAssetsByRuleId = { rule_id: getPrebuiltRuleMock({ rule_id: 'rule_id' }) }; const result = calculateRuleSourceForImport({ rule, - prebuiltRuleAssets, + prebuiltRuleAssetsByRuleId, ruleIdExists: true, }); @@ -67,11 +67,11 @@ describe('calculateRuleSourceForImport', () => { it('calculates as external without customizations if an exact match is found', () => { const rule = getRulesSchemaMock(); rule.rule_id = 'rule_id'; - const prebuiltRuleAssets = [getPrebuiltRuleMock(rule)]; + const prebuiltRuleAssetsByRuleId = { rule_id: getPrebuiltRuleMock(rule) }; const result = calculateRuleSourceForImport({ rule, - prebuiltRuleAssets, + prebuiltRuleAssetsByRuleId, ruleIdExists: true, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 43922b99c1c69..396db123f1295 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -26,16 +26,14 @@ import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset */ export const calculateRuleSourceForImport = ({ rule, - prebuiltRuleAssets, + prebuiltRuleAssetsByRuleId, ruleIdExists, }: { rule: ValidatedRuleToImport; - prebuiltRuleAssets: PrebuiltRuleAsset[]; + prebuiltRuleAssetsByRuleId: Record; ruleIdExists: boolean; }): { ruleSource: RuleSource; immutable: boolean } => { - const assetWithMatchingVersion = prebuiltRuleAssets.find( - (asset) => asset.rule_id === rule.rule_id - ); + const assetWithMatchingVersion = prebuiltRuleAssetsByRuleId[rule.rule_id]; // We convert here so that RuleSource calculation can // continue to deal only with RuleResponses. The fields missing from the // incoming rule are not actually needed for the calculation, but only to diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts index 78cc2b0a2e279..94b9c3fb6a4df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -30,6 +30,7 @@ describe('ruleSourceImporter', () => { ruleAssetsClientMock = createPrebuiltRuleAssetsClientMock(); ruleAssetsClientMock.fetchLatestAssets.mockResolvedValue([{}]); ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([]); + ruleAssetsClientMock.fetchAssetsByVersion.mockResolvedValue([]); ruleToImport = { rule_id: 'rule-1', version: 1 } as RuleToImport; subject = createRuleSourceImporter({ @@ -137,7 +138,7 @@ describe('ruleSourceImporter', () => { expect(calculatorSpy).toHaveBeenCalledTimes(1); expect(calculatorSpy).toHaveBeenCalledWith({ rule, - prebuiltRuleAssets: [expect.objectContaining({ rule_id: 'rule-1' })], + prebuiltRuleAssetsByRuleId: { 'rule-1': expect.objectContaining({ rule_id: 'rule-1' }) }, ruleIdExists: true, }); }); @@ -165,7 +166,7 @@ describe('ruleSourceImporter', () => { expect(calculatorSpy).toHaveBeenCalledTimes(1); expect(calculatorSpy).toHaveBeenCalledWith({ rule, - prebuiltRuleAssets: [expect.objectContaining({ rule_id: 'rule-1' })], + prebuiltRuleAssetsByRuleId: { 'rule-1': expect.objectContaining({ rule_id: 'rule-1' }) }, ruleIdExists: true, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts index 522c67a64f717..b9e577114405e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts @@ -37,7 +37,7 @@ export class RuleSourceImporter implements IRuleSourceImporter { private config: ConfigType; private ruleAssetsClient: IPrebuiltRuleAssetsClient; private latestPackagesInstalled: boolean = false; - private matchingAssets: PrebuiltRuleAsset[] = []; + private matchingAssetsByRuleId: Record = {}; private knownRules: RuleSpecifier[] = []; private availableRuleAssetIds: Set = new Set(); @@ -67,7 +67,7 @@ export class RuleSourceImporter implements IRuleSourceImporter { } this.knownRules = rules.map((rule) => ({ rule_id: rule.rule_id, version: rule.version })); - this.matchingAssets = await this.fetchMatchingAssets(); + this.matchingAssetsByRuleId = await this.fetchMatchingAssetsByRuleId(); this.availableRuleAssetIds = new Set(await this.fetchAvailableRuleAssetIds()); } @@ -82,18 +82,22 @@ export class RuleSourceImporter implements IRuleSourceImporter { return calculateRuleSourceForImport({ rule, - prebuiltRuleAssets: this.matchingAssets, + prebuiltRuleAssetsByRuleId: this.matchingAssetsByRuleId, ruleIdExists: this.availableRuleAssetIds.has(rule.rule_id), }); } - private async fetchMatchingAssets(): Promise { + private async fetchMatchingAssetsByRuleId(): Promise> { this.validateSetupState(); - - return fetchMatchingAssets({ + const matchingAssets = await fetchMatchingAssets({ rules: this.knownRules, ruleAssetsClient: this.ruleAssetsClient, }); + + return matchingAssets.reduce>((map, asset) => { + map[asset.rule_id] = asset; + return map; + }, {}); } private async fetchAvailableRuleAssetIds(): Promise { From b236c844f3cf3eb00e709f2d072dda417b6f8ec2 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 8 Oct 2024 17:11:01 -0500 Subject: [PATCH 112/120] More descriptive parameter name This boolean indicates whether the rule in question is identified as a prebuilt rule, more specifically than just the rule_id existing. --- .../logic/import/calculate_rule_source_for_import.test.ts | 8 ++++---- .../logic/import/calculate_rule_source_for_import.ts | 8 ++++---- .../logic/import/calculate_rule_source_from_asset.test.ts | 8 ++++---- .../logic/import/calculate_rule_source_from_asset.ts | 8 ++++---- .../rule_source_importer/rule_source_importer.test.ts | 4 ++-- .../import/rule_source_importer/rule_source_importer.ts | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts index 27999dda27a4f..e3bfb75c6a88d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -14,7 +14,7 @@ describe('calculateRuleSourceForImport', () => { const result = calculateRuleSourceForImport({ rule: getRulesSchemaMock(), prebuiltRuleAssetsByRuleId: {}, - ruleIdExists: false, + isKnownPrebuiltRule: false, }); expect(result).toEqual({ @@ -32,7 +32,7 @@ describe('calculateRuleSourceForImport', () => { const result = calculateRuleSourceForImport({ rule, prebuiltRuleAssetsByRuleId: {}, - ruleIdExists: true, + isKnownPrebuiltRule: true, }); expect(result).toEqual({ @@ -52,7 +52,7 @@ describe('calculateRuleSourceForImport', () => { const result = calculateRuleSourceForImport({ rule, prebuiltRuleAssetsByRuleId, - ruleIdExists: true, + isKnownPrebuiltRule: true, }); expect(result).toEqual({ @@ -72,7 +72,7 @@ describe('calculateRuleSourceForImport', () => { const result = calculateRuleSourceForImport({ rule, prebuiltRuleAssetsByRuleId, - ruleIdExists: true, + isKnownPrebuiltRule: true, }); expect(result).toEqual({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 396db123f1295..6096261f2c3ec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -19,7 +19,7 @@ import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset * @param rule The rule to be imported * @param prebuiltRuleAssets A list of prebuilt rule assets, which may include * the installed version of the specified prebuilt rule. - * @param ruleIdExists {boolean} Whether the rule's rule_id is available as a + * @param isKnownPrebuiltRule {boolean} Whether the rule's rule_id is available as a * prebuilt asset (independent of the specified version). * * @returns The calculated rule_source and immutable fields for the rule @@ -27,11 +27,11 @@ import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset export const calculateRuleSourceForImport = ({ rule, prebuiltRuleAssetsByRuleId, - ruleIdExists, + isKnownPrebuiltRule, }: { rule: ValidatedRuleToImport; prebuiltRuleAssetsByRuleId: Record; - ruleIdExists: boolean; + isKnownPrebuiltRule: boolean; }): { ruleSource: RuleSource; immutable: boolean } => { const assetWithMatchingVersion = prebuiltRuleAssetsByRuleId[rule.rule_id]; // We convert here so that RuleSource calculation can @@ -42,7 +42,7 @@ export const calculateRuleSourceForImport = ({ const ruleSource = calculateRuleSourceFromAsset({ rule: ruleResponseForImport, assetWithMatchingVersion, - ruleIdExists, + isKnownPrebuiltRule, }); return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts index 7d5b539f357d8..9a2f68479fdea 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.test.ts @@ -14,7 +14,7 @@ describe('calculateRuleSourceFromAsset', () => { const result = calculateRuleSourceFromAsset({ rule: getRulesSchemaMock(), assetWithMatchingVersion: undefined, - ruleIdExists: false, + isKnownPrebuiltRule: false, }); expect(result).toEqual({ @@ -27,7 +27,7 @@ describe('calculateRuleSourceFromAsset', () => { const result = calculateRuleSourceFromAsset({ rule: ruleToImport, assetWithMatchingVersion: undefined, - ruleIdExists: true, + isKnownPrebuiltRule: true, }); expect(result).toEqual({ @@ -46,7 +46,7 @@ describe('calculateRuleSourceFromAsset', () => { version: 1, // version 1 (same version as imported rule) // no other overwrites -> no differences }), - ruleIdExists: true, + isKnownPrebuiltRule: true, }); expect(result).toEqual({ @@ -64,7 +64,7 @@ describe('calculateRuleSourceFromAsset', () => { version: 1, // version 1 (same version as imported rule) name: 'Customized name', // mock a customization }), - ruleIdExists: true, + isKnownPrebuiltRule: true, }); expect(result).toEqual({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts index 09b284de60ddc..4f0caf9b10056 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_from_asset.ts @@ -16,20 +16,20 @@ import { calculateIsCustomized } from '../detection_rules_client/mergers/rule_so * * @param rule The rule for which rule_source is being calculated * @param assetWithMatchingVersion The prebuilt rule asset that matches the specified rule_id and version - * @param ruleIdExists Whether a prebuilt rule with the specified rule_id is currently installed + * @param isKnownPrebuiltRule Whether a prebuilt rule with the specified rule_id is currently installed * * @returns The calculated rule_source */ export const calculateRuleSourceFromAsset = ({ rule, assetWithMatchingVersion, - ruleIdExists, + isKnownPrebuiltRule, }: { rule: RuleResponse; assetWithMatchingVersion: PrebuiltRuleAsset | undefined; - ruleIdExists: boolean; + isKnownPrebuiltRule: boolean; }): RuleSource => { - if (!ruleIdExists) { + if (!isKnownPrebuiltRule) { return { type: 'internal', }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts index 94b9c3fb6a4df..0e7ee793308e2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -139,7 +139,7 @@ describe('ruleSourceImporter', () => { expect(calculatorSpy).toHaveBeenCalledWith({ rule, prebuiltRuleAssetsByRuleId: { 'rule-1': expect.objectContaining({ rule_id: 'rule-1' }) }, - ruleIdExists: true, + isKnownPrebuiltRule: true, }); }); @@ -167,7 +167,7 @@ describe('ruleSourceImporter', () => { expect(calculatorSpy).toHaveBeenCalledWith({ rule, prebuiltRuleAssetsByRuleId: { 'rule-1': expect.objectContaining({ rule_id: 'rule-1' }) }, - ruleIdExists: true, + isKnownPrebuiltRule: true, }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts index b9e577114405e..e43d1cb99861d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts @@ -83,7 +83,7 @@ export class RuleSourceImporter implements IRuleSourceImporter { return calculateRuleSourceForImport({ rule, prebuiltRuleAssetsByRuleId: this.matchingAssetsByRuleId, - ruleIdExists: this.availableRuleAssetIds.has(rule.rule_id), + isKnownPrebuiltRule: this.availableRuleAssetIds.has(rule.rule_id), }); } From 2838173a4d74268e7d8e9231f7bbf6514043a5ac Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 9 Oct 2024 21:45:17 -0500 Subject: [PATCH 113/120] Update x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts Co-authored-by: Georgii Gorbachev --- .../detection_rules_client/converters/normalize_rule_params.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts index b34697c1e0949..7917bc0a10b22 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts @@ -50,7 +50,7 @@ export const normalizeRuleParams = (params: BaseRuleParams): NormalizedRuleParam return { ...params, - // These fields are types as optional in the data model, but they are required in our domain + // These fields are typed as optional in the data model, but they are required in our domain setup: params.setup ?? '', relatedIntegrations: params.relatedIntegrations ?? [], requiredFields: params.requiredFields ?? [], From 86bee40c7eeebb09bf6e0d6704bce15d2855e412 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 9 Oct 2024 21:48:35 -0500 Subject: [PATCH 114/120] Add additional test case for preserving multiple errors per rule --- .../logic/import/import_rules.test.ts | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts index c6951f6b809f6..a6e4e8bad297d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules.test.ts @@ -47,7 +47,7 @@ describe('importRules', () => { message: 'import error', }), createRuleImportErrorObject({ - ruleId: 'rule-id', + ruleId: 'rule-id-2', message: 'import error', }), ]); @@ -72,6 +72,43 @@ describe('importRules', () => { message: 'import error', status_code: 400, }, + rule_id: 'rule-id-2', + }, + ]); + }); + + it('returns multiple errors for the same rule if client import returns generic errors', async () => { + detectionRulesClient.importRules.mockResolvedValueOnce([ + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'import error', + }), + createRuleImportErrorObject({ + ruleId: 'rule-id', + message: 'import error 2', + }), + ]); + + const result = await importRules({ + ruleChunks: [[ruleToImport]], + overwriteRules: false, + detectionRulesClient, + ruleSourceImporter: mockRuleSourceImporter, + }); + + expect(result).toEqual([ + { + error: { + message: 'import error', + status_code: 400, + }, + rule_id: 'rule-id', + }, + { + error: { + message: 'import error 2', + status_code: 400, + }, rule_id: 'rule-id', }, ]); From 6f570fab923b1f288532b5187d08f4b3fc00cfcd Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 9 Oct 2024 21:54:27 -0500 Subject: [PATCH 115/120] Removing general types.ts in favor of a declarative interface file This also makes it more obvious that the `RuleSpecifier` type should be placed with the utility functions that expect them as arguments. --- .../logic/import/rule_source_importer/index.ts | 2 +- .../import/rule_source_importer/rule_source_importer.ts | 4 ++-- .../{types.ts => rule_source_importer_interface.ts} | 5 ----- .../logic/import/rule_source_importer/utils.ts | 6 +++++- 4 files changed, 8 insertions(+), 9 deletions(-) rename x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/{types.ts => rule_source_importer_interface.ts} (89%) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/index.ts index 91087c6b607a1..616c3d28eae95 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/index.ts @@ -5,5 +5,5 @@ * 2.0. */ -export * from './types'; +export * from './rule_source_importer_interface'; export * from './rule_source_importer'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts index e43d1cb99861d..3e6ff990e5dc0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts @@ -22,8 +22,8 @@ import type { PrebuiltRuleAsset } from '../../../../prebuilt_rules'; import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import { ensureLatestRulesPackageInstalled } from '../../../../prebuilt_rules/logic/ensure_latest_rules_package_installed'; import { calculateRuleSourceForImport } from '../calculate_rule_source_for_import'; -import type { CalculatedRuleSource, IRuleSourceImporter, RuleSpecifier } from './types'; -import { fetchAvailableRuleAssetIds, fetchMatchingAssets } from './utils'; +import type { CalculatedRuleSource, IRuleSourceImporter } from './rule_source_importer_interface'; +import { type RuleSpecifier, fetchAvailableRuleAssetIds, fetchMatchingAssets } from './utils'; /** * diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer_interface.ts similarity index 89% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/types.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer_interface.ts index de5e52956ac16..82716cf9413b9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer_interface.ts @@ -11,11 +11,6 @@ import type { ValidatedRuleToImport, } from '../../../../../../../common/api/detection_engine'; -export interface RuleSpecifier { - rule_id: string; - version: number | undefined; -} - export interface CalculatedRuleSource { ruleSource: RuleSource; immutable: boolean; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts index a9e077202b2fc..91ca050d82ca3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts @@ -7,7 +7,11 @@ import type { PrebuiltRuleAsset } from '../../../../prebuilt_rules'; import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; -import type { RuleSpecifier } from './types'; + +export interface RuleSpecifier { + rule_id: string; + version: number | undefined; +} /** * Retrieves the rule IDs (`rule_id`s) of available prebuilt rule assets matching those From 52e1117287fa04d73c157ed1489528aba415d539 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 9 Oct 2024 21:58:05 -0500 Subject: [PATCH 116/120] Simplify interface of RuleSourceImporter Setup's single argument doesn't need to be a named key in an object. --- .../methods/import_rules.ts | 2 +- .../rule_source_importer.test.ts | 26 +++++++++---------- .../rule_source_importer.ts | 2 +- .../rule_source_importer_interface.ts | 2 +- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts index c5ef14ccf12a8..0a66813289290 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/methods/import_rules.ts @@ -43,7 +43,7 @@ export const importRules = async ({ rules, savedObjectsClient, }); - await ruleSourceImporter.setup({ rules }); + await ruleSourceImporter.setup(rules); return Promise.all( rules.map(async (rule) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts index 0e7ee793308e2..39c937f4645a7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.test.ts @@ -50,15 +50,15 @@ describe('ruleSourceImporter', () => { describe('#setup()', () => { it('fetches the rules package on the initial call', async () => { - await subject.setup({ rules: [] }); + await subject.setup([]); expect(ruleAssetsClientMock.fetchLatestAssets).toHaveBeenCalledTimes(1); }); it('does not fetch the rules package on subsequent calls', async () => { - await subject.setup({ rules: [] }); - await subject.setup({ rules: [] }); - await subject.setup({ rules: [] }); + await subject.setup([]); + await subject.setup([]); + await subject.setup([]); expect(ruleAssetsClientMock.fetchLatestAssets).toHaveBeenCalledTimes(1); }); @@ -66,9 +66,7 @@ describe('ruleSourceImporter', () => { it('throws an error if the ruleAsstClient does', async () => { ruleAssetsClientMock.fetchLatestAssets.mockReset().mockRejectedValue(new Error('failed')); - await expect(() => subject.setup({ rules: [] })).rejects.toThrowErrorMatchingInlineSnapshot( - `"failed"` - ); + await expect(() => subject.setup([])).rejects.toThrowErrorMatchingInlineSnapshot(`"failed"`); }); }); @@ -79,26 +77,26 @@ describe('ruleSourceImporter', () => { it("returns false if the rule's rule_id doesn't match an available rule asset", async () => { ruleAssetsClientMock.fetchLatestVersions.mockReset().mockResolvedValue([]); - await subject.setup({ rules: [ruleToImport] }); + await subject.setup([ruleToImport]); expect(subject.isPrebuiltRule(ruleToImport)).toBe(false); }); it("returns true if the rule's rule_id matches an available rule asset", async () => { - await subject.setup({ rules: [ruleToImport] }); + await subject.setup([ruleToImport]); expect(subject.isPrebuiltRule(ruleToImport)).toBe(true); }); it('returns true if the rule has no version, but its rule_id matches an available rule asset', async () => { const ruleWithoutVersion = { ...ruleToImport, version: undefined }; - await subject.setup({ rules: [ruleWithoutVersion] }); + await subject.setup([ruleWithoutVersion]); expect(subject.isPrebuiltRule(ruleWithoutVersion)).toBe(true); }); it('throws an error if the rule is not known to the calculator', async () => { - await subject.setup({ rules: [ruleToImport] }); + await subject.setup([ruleToImport]); expect(() => subject.isPrebuiltRule({ rule_id: 'other-rule' } as RuleToImport) @@ -132,7 +130,7 @@ describe('ruleSourceImporter', () => { }); it('invokes calculateRuleSourceForImport with the correct arguments', async () => { - await subject.setup({ rules: [rule] }); + await subject.setup([rule]); await subject.calculateRuleSource(rule); expect(calculatorSpy).toHaveBeenCalledTimes(1); @@ -145,7 +143,7 @@ describe('ruleSourceImporter', () => { it('throws an error if the rule is not known to the calculator', async () => { ruleAssetsClientMock.fetchLatestVersions.mockResolvedValue([ruleToImport]); - await subject.setup({ rules: [ruleToImport] }); + await subject.setup([ruleToImport]); expect(() => subject.calculateRuleSource(rule)).toThrowErrorMatchingInlineSnapshot( `"Rule validated-rule was not registered during setup."` @@ -160,7 +158,7 @@ describe('ruleSourceImporter', () => { describe('for rules set up without a version', () => { it('invokes the calculator with the correct arguments', async () => { - await subject.setup({ rules: [{ ...rule, version: undefined }] }); + await subject.setup([{ ...rule, version: undefined }]); await subject.calculateRuleSource(rule); expect(calculatorSpy).toHaveBeenCalledTimes(1); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts index 3e6ff990e5dc0..d0972cce9b216 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts @@ -60,7 +60,7 @@ export class RuleSourceImporter implements IRuleSourceImporter { * Prepares the importing of rules by ensuring the latest rules * package is installed and fetching the associated prebuilt rule assets. */ - public async setup({ rules }: { rules: RuleToImport[] }): Promise { + public async setup(rules: RuleToImport[]): Promise { if (!this.latestPackagesInstalled) { await ensureLatestRulesPackageInstalled(this.ruleAssetsClient, this.config, this.context); this.latestPackagesInstalled = true; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer_interface.ts index 82716cf9413b9..ea19672863eeb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer_interface.ts @@ -17,7 +17,7 @@ export interface CalculatedRuleSource { } export interface IRuleSourceImporter { - setup: ({ rules }: { rules: RuleToImport[] }) => Promise; + setup: (rules: RuleToImport[]) => Promise; isPrebuiltRule: (rule: RuleToImport) => boolean; calculateRuleSource: (rule: ValidatedRuleToImport) => CalculatedRuleSource; } From e926f4a5c6d28065e1f0b595dbe0f877cf7d5ec7 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 9 Oct 2024 22:01:07 -0500 Subject: [PATCH 117/120] Inline our utils.ts functions Since a "utility" file is prone to collect random things, and these functions are all specific to the importer itself, we're now inlining them into the same file as the class, until such time as they need to be reused. --- .../rule_source_importer.ts | 61 +++++++++++++++- .../import/rule_source_importer/utils.ts | 69 ------------------- 2 files changed, 60 insertions(+), 70 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts index d0972cce9b216..1f5c2c5aa543b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/rule_source_importer.ts @@ -23,7 +23,66 @@ import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic import { ensureLatestRulesPackageInstalled } from '../../../../prebuilt_rules/logic/ensure_latest_rules_package_installed'; import { calculateRuleSourceForImport } from '../calculate_rule_source_for_import'; import type { CalculatedRuleSource, IRuleSourceImporter } from './rule_source_importer_interface'; -import { type RuleSpecifier, fetchAvailableRuleAssetIds, fetchMatchingAssets } from './utils'; + +interface RuleSpecifier { + rule_id: string; + version: number | undefined; +} + +/** + * Retrieves the rule IDs (`rule_id`s) of available prebuilt rule assets matching those + * of the specified rules. This information can be used to determine whether + * the rule being imported is a custom rule or a prebuilt rule. + * + * @param rules - A list of {@link RuleSpecifier}s representing the rules being imported. + * @param ruleAssetsClient - the {@link IPrebuiltRuleAssetsClient} to use for fetching the available rule assets. + * + * @returns A list of the prebuilt rule asset IDs that are available. + * + */ +const fetchAvailableRuleAssetIds = async ({ + rules, + ruleAssetsClient, +}: { + rules: RuleSpecifier[]; + ruleAssetsClient: IPrebuiltRuleAssetsClient; +}): Promise => { + const incomingRuleIds = rules.map((rule) => rule.rule_id); + const availableRuleAssetSpecifiers = await ruleAssetsClient.fetchLatestVersions(incomingRuleIds); + + return availableRuleAssetSpecifiers.map((specifier) => specifier.rule_id); +}; + +/** + * Retrieves prebuilt rule assets for rules being imported. These + * assets can be compared to the incoming rules for the purposes of calculating + * appropriate `rule_source` values. + * + * @param rules - A list of {@link RuleSpecifier}s representing the rules being imported. + * + * @returns The prebuilt rule assets matching the specified prebuilt + * rules. Assets match the `rule_id` and `version` of the specified rules. + * Because of this, there may be less assets returned than specified rules. + */ +const fetchMatchingAssets = async ({ + rules, + ruleAssetsClient, +}: { + rules: RuleSpecifier[]; + ruleAssetsClient: IPrebuiltRuleAssetsClient; +}): Promise => { + const incomingRuleVersions = rules.flatMap((rule) => { + if (rule.version == null) { + return []; + } + return { + rule_id: rule.rule_id, + version: rule.version, + }; + }); + + return ruleAssetsClient.fetchAssetsByVersion(incomingRuleVersions); +}; /** * diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts deleted file mode 100644 index 91ca050d82ca3..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/rule_source_importer/utils.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { PrebuiltRuleAsset } from '../../../../prebuilt_rules'; -import type { IPrebuiltRuleAssetsClient } from '../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; - -export interface RuleSpecifier { - rule_id: string; - version: number | undefined; -} - -/** - * Retrieves the rule IDs (`rule_id`s) of available prebuilt rule assets matching those - * of the specified rules. This information can be used to determine whether - * the rule being imported is a custom rule or a prebuilt rule. - * - * @param rules - A list of {@link RuleSpecifier}s representing the rules being imported. - * @param ruleAssetsClient - the {@link IPrebuiltRuleAssetsClient} to use for fetching the available rule assets. - * - * @returns A list of the prebuilt rule asset IDs that are available. - * - */ -export const fetchAvailableRuleAssetIds = async ({ - rules, - ruleAssetsClient, -}: { - rules: RuleSpecifier[]; - ruleAssetsClient: IPrebuiltRuleAssetsClient; -}): Promise => { - const incomingRuleIds = rules.map((rule) => rule.rule_id); - const availableRuleAssetSpecifiers = await ruleAssetsClient.fetchLatestVersions(incomingRuleIds); - - return availableRuleAssetSpecifiers.map((specifier) => specifier.rule_id); -}; - -/** - * Retrieves prebuilt rule assets for rules being imported. These - * assets can be compared to the incoming rules for the purposes of calculating - * appropriate `rule_source` values. - * - * @param rules - A list of {@link RuleSpecifier}s representing the rules being imported. - * - * @returns The prebuilt rule assets matching the specified prebuilt - * rules. Assets match the `rule_id` and `version` of the specified rules. - * Because of this, there may be less assets returned than specified rules. - */ -export const fetchMatchingAssets = async ({ - rules, - ruleAssetsClient, -}: { - rules: RuleSpecifier[]; - ruleAssetsClient: IPrebuiltRuleAssetsClient; -}): Promise => { - const incomingRuleVersions = rules.flatMap((rule) => { - if (rule.version == null) { - return []; - } - return { - rule_id: rule.rule_id, - version: rule.version, - }; - }); - - return ruleAssetsClient.fetchAssetsByVersion(incomingRuleVersions); -}; From 6062161d2398d348fa85628d1ea696b3146cd5a1 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 10 Oct 2024 17:07:41 -0500 Subject: [PATCH 118/120] Add proper defaults to rule during import Since `calculateIsCustomized` will fail if the provided rule is not a valid RuleResponse, we were experiencing failures (uncovered by integration tests) due to a improper conversion of incoming rules. I _believe_ that I've fixed this by: 1. Defining an explicit converter utility for this case: RuleToImport -> RuleResponse 2. Calling `applyRuleDefaults` within this utility `applyRuleDefaults` is what, in the normal import code path, adds these missing/default fields. My assumption is that this will cover any other issues with conversion to a `RuleResponse`, which will ideally be validated by the Rules Management team. The same issue still exists for the previously-used converter utility, but I'm going to tackle that in a separate commit. --- .../calculate_rule_source_for_import.ts | 4 +-- ...rt_rule_to_import_to_rule_response.test.ts | 33 +++++++++++++++++++ ...convert_rule_to_import_to_rule_response.ts | 27 +++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 6096261f2c3ec..133566a7b776b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -10,8 +10,8 @@ import type { ValidatedRuleToImport, } from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; -import { convertPrebuiltRuleAssetToRuleResponse } from '../detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response'; import { calculateRuleSourceFromAsset } from './calculate_rule_source_from_asset'; +import { convertRuleToImportToRuleResponse } from './converters/convert_rule_to_import_to_rule_response'; /** * Calculates the rule_source field for a rule being imported @@ -38,7 +38,7 @@ export const calculateRuleSourceForImport = ({ // continue to deal only with RuleResponses. The fields missing from the // incoming rule are not actually needed for the calculation, but only to // satisfy the type system. - const ruleResponseForImport = convertPrebuiltRuleAssetToRuleResponse(rule); + const ruleResponseForImport = convertRuleToImportToRuleResponse(rule); const ruleSource = calculateRuleSourceFromAsset({ rule: ruleResponseForImport, assetWithMatchingVersion, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.test.ts new file mode 100644 index 0000000000000..bd486764576de --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.test.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + getImportRulesSchemaMock, + getValidatedRuleToImportMock, +} from '../../../../../../../common/api/detection_engine/rule_management/mocks'; +import { convertRuleToImportToRuleResponse } from './convert_rule_to_import_to_rule_response'; + +describe('convertRuleToImportToRuleResponse', () => { + it('converts a valid RuleToImport (without a language field) to valid RuleResponse (with a language field)', () => { + const ruleToImportWithoutLanguage = getImportRulesSchemaMock({ language: undefined }); + + expect(convertRuleToImportToRuleResponse(ruleToImportWithoutLanguage)).toMatchObject({ + language: 'kuery', + rule_id: ruleToImportWithoutLanguage.rule_id, + }); + }); + + it('converts a ValidatedRuleToImport and preserves its version', () => { + const ruleToImport = getValidatedRuleToImportMock({ version: 99 }); + + expect(convertRuleToImportToRuleResponse(ruleToImport)).toMatchObject({ + version: 99, + language: 'kuery', + rule_id: ruleToImport.rule_id, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.ts new file mode 100644 index 0000000000000..cbc8826c4b078 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/converters/convert_rule_to_import_to_rule_response.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { v4 as uuidv4 } from 'uuid'; +import { applyRuleDefaults } from '../../detection_rules_client/mergers/apply_rule_defaults'; +import { RuleResponse, type RuleToImport } from '../../../../../../../common/api/detection_engine'; + +export const convertRuleToImportToRuleResponse = (ruleToImport: RuleToImport): RuleResponse => { + const ruleResponseSpecificFields = { + id: uuidv4(), + updated_at: new Date().toISOString(), + updated_by: '', + created_at: new Date().toISOString(), + created_by: '', + revision: 1, + }; + const ruleWithDefaults = applyRuleDefaults(ruleToImport); + + return RuleResponse.parse({ + ...ruleResponseSpecificFields, + ...ruleWithDefaults, + }); +}; From 3feb6936a4e60850be651aa246c3ec1b43b0bc28 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 10 Oct 2024 17:36:27 -0500 Subject: [PATCH 119/120] Define RuleToImport mocks with minimal fields We don't want to specify `language` here since it's not a required field, and their presence could actually mask bugs with defaulting logic throughout our test suite. I believe there were just a few tests that were (unnecessarily) dependent on those fields; everything else luckily just works. --- .../rule_management/import_rules/rule_to_import.mock.ts | 3 --- .../action_connectors/import_rule_action_connectors.test.ts | 3 --- 2 files changed, 6 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts index 2d98cace04b2b..6807f50d3f6b1 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.mock.ts @@ -15,7 +15,6 @@ export const getImportRulesSchemaMock = (rewrites?: Partial): Rule severity: 'high', type: 'query', risk_score: 55, - language: 'kuery', rule_id: 'rule-1', immutable: false, ...rewrites, @@ -36,7 +35,6 @@ export const getImportRulesWithIdSchemaMock = (ruleId = 'rule-1'): RuleToImport severity: 'high', type: 'query', risk_score: 55, - language: 'kuery', rule_id: ruleId, immutable: false, }); @@ -70,7 +68,6 @@ export const getImportThreatMatchRulesSchemaMock = ( severity: 'high', type: 'threat_match', risk_score: 55, - language: 'kuery', rule_id: 'rule-1', threat_index: ['index-123'], threat_mapping: [{ entries: [{ field: 'host.name', type: 'mapping', value: 'host.name' }] }], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts index 84352c1ea0f1e..08bfa95207555 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts @@ -373,7 +373,6 @@ describe('importRuleActionConnectors', () => { ], description: 'some description', immutable: false, - language: 'kuery', name: 'Query with a rule id', query: 'user.name: root or user.name: admin', risk_score: 55, @@ -483,7 +482,6 @@ describe('importRuleActionConnectors', () => { ], description: 'some description', immutable: false, - language: 'kuery', name: 'Query with a rule id', query: 'user.name: root or user.name: admin', risk_score: 55, @@ -502,7 +500,6 @@ describe('importRuleActionConnectors', () => { ], description: 'some description', immutable: false, - language: 'kuery', name: 'Query with a rule id', id: '0abc78e0-7031-11ed-b076-53cc4d57aaf1', rule_id: 'rule_2', From 4ef7a394d7e169742e1f7798671655ae04df6dc5 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 10 Oct 2024 17:44:24 -0500 Subject: [PATCH 120/120] Fix bug converting a prebuilt rule asset to a rule response If a prebuilt asset is lacking a 'language' field, this conversion function would previously throw an error due to us not providing a default for it. This was fixed by calling `applyRuleDefaults` within this function, which made some other logic in this function redundant (and so it was removed). --- ...ebuilt_rule_asset_to_rule_response.test.ts | 19 +++++++++++++++++++ ...rt_prebuilt_rule_asset_to_rule_response.ts | 9 ++++----- 2 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.test.ts new file mode 100644 index 0000000000000..98e5b235159e8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.test.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { convertPrebuiltRuleAssetToRuleResponse } from './convert_prebuilt_rule_asset_to_rule_response'; +import { getPrebuiltRuleMock } from '../../../../prebuilt_rules/mocks'; + +describe('convertPrebuiltRuleAssetToRuleResponse', () => { + it('converts a valid prebuilt asset (without a language field) to valid rule response (with a language field)', () => { + const ruleAssetWithoutLanguage = getPrebuiltRuleMock({ language: undefined }); + + expect(convertPrebuiltRuleAssetToRuleResponse(ruleAssetWithoutLanguage)).toMatchObject({ + language: 'kuery', + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts index f7a5d78798880..1e87721557214 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts @@ -6,10 +6,9 @@ */ import { v4 as uuidv4 } from 'uuid'; -import { addEcsToRequiredFields } from '../../../../../../../common/detection_engine/rule_management/utils'; import { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; import type { PrebuiltRuleAsset } from '../../../../prebuilt_rules'; -import { RULE_DEFAULTS } from '../mergers/apply_rule_defaults'; +import { applyRuleDefaults } from '../mergers/apply_rule_defaults'; export const convertPrebuiltRuleAssetToRuleResponse = ( prebuiltRuleAsset: PrebuiltRuleAsset @@ -30,10 +29,10 @@ export const convertPrebuiltRuleAssetToRuleResponse = ( revision: 1, }; + const ruleWithDefaults = applyRuleDefaults(prebuiltRuleAsset); + return RuleResponse.parse({ - ...RULE_DEFAULTS, - ...prebuiltRuleAsset, - required_fields: addEcsToRequiredFields(prebuiltRuleAsset.required_fields), + ...ruleWithDefaults, ...ruleResponseSpecificFields, }); };