[Security Solution][Detections] Allows rules to be updated from EPR

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -36,7 +36,7 @@ export type EpmPackageInstallStatus = 'installed' | 'installing';
 export type DetailViewPanelName = 'overview' | 'policies' | 'settings' | 'custom';
 export type ServiceName = 'kibana' | 'elasticsearch';
 export type AgentAssetType = typeof agentAssetTypes;
-export type AssetType = KibanaAssetType | ElasticsearchAssetType | ValueOf<AgentAssetType>;
+export type AssetType = KibanaAssetType | ElasticsearchAssetType | ValueOf<AgentAssetType> | 'rules';
 
 /*
   Enum mapping of a saved object asset type to how it would appear in a package file path (snake cased)

x-pack/plugins/fleet/public/applications/fleet/sections/epm/constants.tsx

@@ -31,6 +31,7 @@ export const AssetTitleMap: Record<AssetType, string> = {
   map: 'Map',
   data_stream_ilm_policy: 'Data Stream ILM Policy',
   lens: 'Lens',
+  rules: 'Detection Rules'
 };
 
 export const ServiceTitleMap: Record<ServiceName, string> = {

x-pack/plugins/fleet/server/index.ts

@@ -22,6 +22,8 @@ export {
   PackageService,
   AgentPolicyServiceInterface,
 } from './services';
+export { getRegistryPackage, fetchFindLatestPackage } from './services/epm/registry';
+export { getAsset, getPathParts } from './services/epm/archive';
 export { FleetSetupContract, FleetSetupDeps, FleetStartContract, ExternalCallback } from './plugin';
 
 export const config: PluginConfigDescriptor = {

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -520,7 +520,7 @@ export const success_count = PositiveInteger;
 export const rules_custom_installed = PositiveInteger;
 export const rules_not_installed = PositiveInteger;
 export const rules_not_updated = PositiveInteger;
-
+export const rules_package_version = t.union([t.string, t.undefined]);
 export const timelines_installed = PositiveInteger;
 export const timelines_updated = PositiveInteger;
 export const timelines_not_installed = PositiveInteger;

x-pack/plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.ts

@@ -67,6 +67,7 @@ import {
   throttleOrNull,
   createdByOrNull,
   updatedByOrNull,
+  rules_package_version,
 } from '../common/schemas';
 
 const createSchema = <
@@ -406,6 +407,11 @@ const responseTypeSpecific = t.union([
 ]);
 export type ResponseTypeSpecific = t.TypeOf<typeof responseTypeSpecific>;
 
+export const installPrepackagedRulesSchema = t.type({
+  rules_package_version,
+});
+export type InstallPrepackagedRulesSchema = t.TypeOf<typeof installPrepackagedRulesSchema>;
+
 export const updateRulesSchema = t.intersection([
   commonCreateParams,
   createTypeSpecific,

x-pack/plugins/security_solution/common/detection_engine/schemas/response/prepackaged_rules_status_schema.ts

@@ -12,6 +12,7 @@ import {
   rules_custom_installed,
   rules_not_installed,
   rules_not_updated,
+  rules_package_version,
   timelines_installed,
   timelines_not_installed,
   timelines_not_updated,
@@ -30,8 +31,16 @@ const prePackagedRulesStatusSchema = t.type({
   rules_not_updated,
 });
 
+const prePackagedRulesRegistrySchema = t.type({
+  rules_package_version,
+});
+
 export const prePackagedRulesAndTimelinesStatusSchema = t.exact(
-  t.intersection([prePackagedRulesStatusSchema, prePackagedTimelinesStatusSchema])
+  t.intersection([
+    prePackagedRulesStatusSchema,
+    prePackagedTimelinesStatusSchema,
+    prePackagedRulesRegistrySchema,
+  ])
 );
 
 export type PrePackagedRulesAndTimelinesStatusSchema = t.TypeOf<

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.ts

@@ -24,7 +24,6 @@ import {
   FetchRulesResponse,
   Rule,
   FetchRuleProps,
-  BasicFetchProps,
   ImportDataProps,
   ExportDocumentsProps,
   RuleStatusResponse,
@@ -246,25 +245,32 @@ export const duplicateRules = async ({ rules }: DuplicateRulesProps): Promise<Bu
 /**
  * Create Prepackaged Rules
  *
+ * @param pkgVersion Optional version of the rules package to install
  * @param signal AbortSignal for cancelling request
  *
  * @throws An error if response is not OK
  */
 export const createPrepackagedRules = async ({
+  pkgVersion,
   signal,
-}: BasicFetchProps): Promise<{
+}: {
+  pkgVersion: string | undefined | null;
+  signal: AbortSignal;
+}): Promise<{
   rules_installed: number;
   rules_updated: number;
   timelines_installed: number;
   timelines_updated: number;
 }> => {
+  const body = pkgVersion ? { rules_package_version: pkgVersion } : {};
   const result = await KibanaServices.get().http.fetch<{
     rules_installed: number;
     rules_updated: number;
     timelines_installed: number;
     timelines_updated: number;
   }>(DETECTION_ENGINE_PREPACKAGED_URL, {
     method: 'PUT',
+    body: JSON.stringify(body),
     signal,
   });
 

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts

@@ -283,6 +283,7 @@ export interface PrePackagedRulesStatusResponse {
   rules_installed: number;
   rules_not_installed: number;
   rules_not_updated: number;
+  rules_package_version: string | null | undefined;
   timelines_installed: number;
   timelines_not_installed: number;
   timelines_not_updated: number;

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_pre_packaged_rules.tsx

@@ -56,6 +56,7 @@ interface ReturnPrePackagedRules {
   rulesInstalled: number | null;
   rulesNotInstalled: number | null;
   rulesNotUpdated: number | null;
+  rulesPackageVersion: string | null | undefined;
   getLoadPrebuiltRulesAndTemplatesButton: GetLoadPrebuiltRulesAndTemplatesButton;
   getReloadPrebuiltRulesAndTemplatesButton: GetReloadPrebuiltRulesAndTemplatesButton;
 }
@@ -96,6 +97,7 @@ export const usePrePackagedRules = ({
       | 'rulesInstalled'
       | 'rulesNotInstalled'
       | 'rulesNotUpdated'
+      | 'rulesPackageVersion'
       | 'timelinesInstalled'
       | 'timelinesNotInstalled'
       | 'timelinesNotUpdated'
@@ -107,6 +109,7 @@ export const usePrePackagedRules = ({
     rulesInstalled: null,
     rulesNotInstalled: null,
     rulesNotUpdated: null,
+    rulesPackageVersion: null,
     timelinesInstalled: null,
     timelinesNotInstalled: null,
     timelinesNotUpdated: null,
@@ -148,12 +151,14 @@ export const usePrePackagedRules = ({
         });
         if (isSubscribed) {
           setPrepackagedDataStatus({
-            createPrePackagedRules: createElasticRules,
+            createPrePackagedRules: async () =>
+              createElasticRules(prePackagedRuleStatusResponse.rules_package_version),
             refetchPrePackagedRulesStatus: fetchPrePackagedRules,
             rulesCustomInstalled: prePackagedRuleStatusResponse.rules_custom_installed,
             rulesInstalled: prePackagedRuleStatusResponse.rules_installed,
             rulesNotInstalled: prePackagedRuleStatusResponse.rules_not_installed,
             rulesNotUpdated: prePackagedRuleStatusResponse.rules_not_updated,
+            rulesPackageVersion: prePackagedRuleStatusResponse.rules_package_version,
             timelinesInstalled: prePackagedRuleStatusResponse.timelines_installed,
             timelinesNotInstalled: prePackagedRuleStatusResponse.timelines_not_installed,
             timelinesNotUpdated: prePackagedRuleStatusResponse.timelines_not_updated,
@@ -168,6 +173,7 @@ export const usePrePackagedRules = ({
             rulesInstalled: null,
             rulesNotInstalled: null,
             rulesNotUpdated: null,
+            rulesPackageVersion: null,
             timelinesInstalled: null,
             timelinesNotInstalled: null,
             timelinesNotUpdated: null,
@@ -181,7 +187,7 @@ export const usePrePackagedRules = ({
       }
     };
 
-    const createElasticRules = async (): Promise<boolean> => {
+    const createElasticRules = async (pkgVersion: string | null | undefined): Promise<boolean> => {
       return new Promise(async (resolve) => {
         try {
           if (
@@ -193,6 +199,7 @@ export const usePrePackagedRules = ({
           ) {
             setLoadingCreatePrePackagedRules(true);
             const result = await createPrepackagedRules({
+              pkgVersion,
               signal: abortCtrl.signal,
             });
 
@@ -221,12 +228,14 @@ export const usePrePackagedRules = ({
                   ) {
                     setLoadingCreatePrePackagedRules(false);
                     setPrepackagedDataStatus({
-                      createPrePackagedRules: createElasticRules,
+                      createPrePackagedRules: async () =>
+                        createElasticRules(prePackagedRuleStatusResponse.rules_package_version),
                       refetchPrePackagedRulesStatus: fetchPrePackagedRules,
                       rulesCustomInstalled: prePackagedRuleStatusResponse.rules_custom_installed,
                       rulesInstalled: prePackagedRuleStatusResponse.rules_installed,
                       rulesNotInstalled: prePackagedRuleStatusResponse.rules_not_installed,
                       rulesNotUpdated: prePackagedRuleStatusResponse.rules_not_updated,
+                      rulesPackageVersion: prePackagedRuleStatusResponse.rules_package_version,
                       timelinesInstalled: prePackagedRuleStatusResponse.timelines_installed,
                       timelinesNotInstalled: prePackagedRuleStatusResponse.timelines_not_installed,
                       timelinesNotUpdated: prePackagedRuleStatusResponse.timelines_not_updated,

x-pack/plugins/security_solution/server/config.ts

@@ -35,6 +35,7 @@ export const configSchema = schema.object({
    */
   packagerTaskInterval: schema.string({ defaultValue: '60s' }),
   validateArtifactDownloads: schema.boolean({ defaultValue: true }),
+  usePackageRegistryRules: schema.boolean({ defaultValue: false }),
 });
 
 export const createConfig = (context: PluginInitializerContext) =>

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts

@@ -12,6 +12,11 @@ import type {
 } from '../../../../types';
 
 import { validate } from '../../../../../common/validate';
+import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
+import {
+  installPrepackagedRulesSchema,
+  InstallPrepackagedRulesSchema,
+} from '../../../../../common/detection_engine/schemas/request/rule_schemas';
 import {
   PrePackagedRulesAndTimelinesSchema,
   prePackagedRulesAndTimelinesSchema,
@@ -25,7 +30,7 @@ import { buildFrameworkRequest } from '../../../timeline/routes/utils/common';
 import { installPrepackagedTimelines } from '../../../timeline/routes/utils/install_prepacked_timelines';
 
 import { getIndexExists } from '../../index/get_index_exists';
-import { getPrepackagedRules } from '../../rules/get_prepackaged_rules';
+import { getRegistryOrFileSystemRules } from '../../rules/get_prepackaged_rules';
 import { installPrepackagedRules } from '../../rules/install_prepacked_rules';
 import { updatePrepackagedRules } from '../../rules/update_prepacked_rules';
 import { getRulesToInstall } from '../../rules/get_rules_to_install';
@@ -46,14 +51,19 @@ export const addPrepackedRulesRoute = (
   router.put(
     {
       path: DETECTION_ENGINE_PREPACKAGED_URL,
-      validate: false,
+      validate: {
+        body: buildRouteValidation<
+          typeof installPrepackagedRulesSchema,
+          InstallPrepackagedRulesSchema
+        >(installPrepackagedRulesSchema),
+      },
       options: {
         tags: ['access:securitySolution'],
       },
     },
-    async (context, _, response) => {
+    async (context, request, response) => {
       const siemResponse = buildSiemResponse(response);
-      const frameworkRequest = await buildFrameworkRequest(context, security, _);
+      const frameworkRequest = await buildFrameworkRequest(context, security, request);
 
       try {
         const alertsClient = context.alerting?.getAlertsClient();
@@ -68,7 +78,9 @@ export const addPrepackedRulesRoute = (
           siemClient,
           alertsClient,
           frameworkRequest,
-          config.maxTimelineImportExportSize
+          config.maxTimelineImportExportSize,
+          undefined,
+          request.body.rules_package_version
         );
         return response.ok({ body: validated ?? {} });
       } catch (err) {
@@ -96,7 +108,8 @@ export const createPrepackagedRules = async (
   alertsClient: AlertsClient,
   frameworkRequest: FrameworkRequest,
   maxTimelineImportExportSize: number,
-  exceptionsClient?: ExceptionListClient
+  exceptionsClient?: ExceptionListClient,
+  rulesPackageVersion?: string
 ): Promise<PrePackagedRulesAndTimelinesSchema | null> => {
   const clusterClient = context.core.elasticsearch.legacy.client;
   const savedObjectsClient = context.core.savedObjects.client;
@@ -112,10 +125,10 @@ export const createPrepackagedRules = async (
     await exceptionsListClient.createEndpointList();
   }
 
-  const rulesFromFileSystem = getPrepackagedRules();
-  const prepackagedRules = await getExistingPrepackagedRules({ alertsClient });
-  const rulesToInstall = getRulesToInstall(rulesFromFileSystem, prepackagedRules);
-  const rulesToUpdate = getRulesToUpdate(rulesFromFileSystem, prepackagedRules);
+  const prepackagedRules = await getRegistryOrFileSystemRules(rulesPackageVersion);
+  const existingPrepackagedRules = await getExistingPrepackagedRules({ alertsClient });
+  const rulesToInstall = getRulesToInstall(prepackagedRules, existingPrepackagedRules);
+  const rulesToUpdate = getRulesToUpdate(prepackagedRules, existingPrepackagedRules);
   const signalsIndex = siemClient.getSignalsIndex();
   if (rulesToInstall.length !== 0 || rulesToUpdate.length !== 0) {
     const signalsIndexExists = await getIndexExists(clusterClient.callAsCurrentUser, signalsIndex);

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts

@@ -13,7 +13,10 @@ import {
 import type { SecuritySolutionPluginRouter } from '../../../../types';
 import { DETECTION_ENGINE_PREPACKAGED_URL } from '../../../../../common/constants';
 import { transformError, buildSiemResponse } from '../utils';
-import { getPrepackagedRules } from '../../rules/get_prepackaged_rules';
+import {
+  getLatestRulesPackageVersion,
+  getRegistryOrFileSystemRules,
+} from '../../rules/get_prepackaged_rules';
 import { getRulesToInstall } from '../../rules/get_rules_to_install';
 import { getRulesToUpdate } from '../../rules/get_rules_to_update';
 import { findRules } from '../../rules/find_rules';
@@ -24,7 +27,7 @@ import { SetupPlugins } from '../../../../plugin';
 import { checkTimelinesStatus } from '../../../timeline/routes/utils/check_timelines_status';
 import { checkTimelineStatusRt } from '../../../timeline/routes/schemas/check_timelines_status_schema';
 
-export const getPrepackagedRulesStatusRoute = (
+export const getPrepackagedRulesStatusRoute = async (
   router: SecuritySolutionPluginRouter,
   config: ConfigType,
   security: SetupPlugins['security']
@@ -46,7 +49,8 @@ export const getPrepackagedRulesStatusRoute = (
       }
 
       try {
-        const rulesFromFileSystem = getPrepackagedRules();
+        const pkgVersion = getLatestRulesPackageVersion();
+        const rulesFromFileSystem = await getRegistryOrFileSystemRules(pkgVersion);
         const customRules = await findRules({
           alertsClient,
           perPage: 1,
@@ -72,6 +76,7 @@ export const getPrepackagedRulesStatusRoute = (
           rules_installed: prepackagedRules.length,
           rules_not_installed: rulesToInstall.length,
           rules_not_updated: rulesToUpdate.length,
+          rules_package_version: pkgVersion,
           timelines_installed: validatedprepackagedTimelineStatus?.prepackagedTimelines.length ?? 0,
           timelines_not_installed:
             validatedprepackagedTimelineStatus?.timelinesToInstall.length ?? 0,

x-pack/plugins/security_solution/server/lib/detection_engine/rules/constants.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+const DETECTION_RULES_PACAKGE_NAME = 'detection_rules';

x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts

@@ -5,17 +5,17 @@
  * 2.0.
  */
 
-import { getPrepackagedRules } from './get_prepackaged_rules';
+import { getFileSystemRules } from './get_prepackaged_rules';
 import { isEmpty } from 'lodash/fp';
 import { AddPrepackagedRulesSchemaDecoded } from '../../../../common/detection_engine/schemas/request/add_prepackaged_rules_schema';
 
-describe('get_existing_prepackaged_rules', () => {
-  test('should not throw any errors with the existing checked in pre-packaged rules', () => {
-    expect(() => getPrepackagedRules()).not.toThrow();
+describe('get_existing_prepackaged_rules', async () => {
+  test('should not throw any errors with the existing checked in pre-packaged rules', async () => {
+    expect(async () => getFileSystemRules()).not.toThrow();
   });
 
-  test('no rule should have the same rule_id as another rule_id', () => {
-    const prePackagedRules = getPrepackagedRules();
+  test('no rule should have the same rule_id as another rule_id', async () => {
+    const prePackagedRules = await getFileSystemRules();
     let existingRuleIds: AddPrepackagedRulesSchemaDecoded[] = [];
     prePackagedRules.forEach((rule) => {
       const foundDuplicate = existingRuleIds.reduce((accum, existingRule) => {
@@ -33,16 +33,18 @@ describe('get_existing_prepackaged_rules', () => {
     });
   });
 
-  test('should throw an exception if a pre-packaged rule is not valid', () => {
+  test('should throw an exception if a pre-packaged rule is not valid', async () => {
     // @ts-expect-error intentionally invalid argument
-    expect(() => getPrepackagedRules([{ not_valid_made_up_key: true }])).toThrow(
+    expect(async () => getFileSystemRules([{ not_valid_made_up_key: true }])).toThrow(
       'name: "(rule name unknown)", rule_id: "(rule rule_id unknown)" within the folder rules/prepackaged_rules is not a valid detection engine rule. Expect the system to not work with pre-packaged rules until this rule is fixed or the file is removed. Error is: Invalid value "undefined" supplied to "description",Invalid value "undefined" supplied to "risk_score",Invalid value "undefined" supplied to "name",Invalid value "undefined" supplied to "severity",Invalid value "undefined" supplied to "type",Invalid value "undefined" supplied to "rule_id",Invalid value "undefined" supplied to "version", Full rule contents are:\n{\n  "not_valid_made_up_key": true\n}'
     );
   });
 
   test('should throw an exception with a message having rule_id and name in it', () => {
-    // @ts-expect-error intentionally invalid argument
-    expect(() => getPrepackagedRules([{ name: 'rule name', rule_id: 'id-123' }])).toThrow(
+    expect(
+      // @ts-expect-error intentionally invalid argument
+      async () => getFileSystemRules([{ name: 'rule name', rule_id: 'id-123' }])
+    ).toThrow(
       'name: "rule name", rule_id: "id-123" within the folder rules/prepackaged_rules is not a valid detection engine rule. Expect the system to not work with pre-packaged rules until this rule is fixed or the file is removed. Error is: Invalid value "undefined" supplied to "description",Invalid value "undefined" supplied to "risk_score",Invalid value "undefined" supplied to "severity",Invalid value "undefined" supplied to "type",Invalid value "undefined" supplied to "version", Full rule contents are:\n{\n  "name": "rule name",\n  "rule_id": "id-123"\n}'
     );
   });