[Security Solution] Prebuilt rule upgrade and installation endpoints,…

… initial implementation (elastic#148392)

**Addresses:** elastic#148181,
elastic#148182,
elastic#148185
**Partially addresses:**
elastic#148183,
elastic#148189

## Summary

Based on the [POC](elastic#144060), this
PR adds 4 endpoints for the new upgrade and installation workflows for
prebuilt rules:

- `GET /internal/detection_engine/prebuilt_rules/status`
- `POST /internal/detection_engine/prebuilt_rules/upgrade/_review`
- `POST /internal/detection_engine/prebuilt_rules/installation/_review`
- `POST /internal/detection_engine/prebuilt_rules/_generate_assets`
(temporary helper endpoint for development and testing)

The new endpoints are hidden behind a feature flag and can be enabled by
the following config setting:

```yaml
xpack.securitySolution.enableExperimental: ['prebuiltRulesNewUpgradeAndInstallationWorkflowsEnabled']
```

## In the next episodes

Will be done later in follow-up PRs:

- Implementation of some additional response properties for the
`upgrade/_review` endpoint:
  - elastic#148183
- Making base versions optional for diff calculation (we need to support
this in order to be able to still show diffs for rule assets coming from
packages without historical versions):
  - elastic#148189
- Further development of the diff algorithm:
  - elastic#148191
- Test coverage:
  - elastic#148192

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/get_prebuilt_rules_status/response_schema.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface GetPrebuiltRulesStatusResponseBody {
+  status_code: number;
+  message: string;
+  attributes: {
+    /** Aggregated info about all prebuilt rules */
+    stats: PrebuiltRulesStatusStats;
+  };
+}
+
+export interface PrebuiltRulesStatusStats {
+  /** Total number of existing (known) prebuilt rules */
+  num_prebuilt_rules_total: number;
+
+  /** Number of installed prebuilt rules */
+  num_prebuilt_rules_installed: number;
+
+  /** Number of prebuilt rules available for installation (not yet installed) */
+  num_prebuilt_rules_to_install: number;
+
+  /** Number of installed prebuilt rules available for upgrade (stock + customized) */
+  num_prebuilt_rules_to_upgrade: number;
+
+  // In the future we could add more stats such as:
+  // - number of installed prebuilt rules which were deprecated
+  // - number of installed prebuilt rules which are not compatible with the current version of Kibana
+}

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/review_rule_installation/response_schema.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { RuleSignatureId, RuleTagArray, RuleVersion } from '../../../rule_schema';
+import type { DiffableRule } from '../../model/diff/diffable_rule/diffable_rule';
+
+export interface ReviewRuleInstallationResponseBody {
+  status_code: number;
+  message: string;
+  attributes: {
+    /** Aggregated info about all rules available for installation */
+    stats: RuleInstallationStatsForReview;
+
+    /** Info about individual rules: one object per each rule available for installation */
+    rules: RuleInstallationInfoForReview[];
+  };
+}
+
+export interface RuleInstallationStatsForReview {
+  /** Number of prebuilt rules available for installation */
+  num_rules_to_install: number;
+
+  /** A union of all tags of all rules available for installation */
+  tags: RuleTagArray;
+}
+
+export type RuleInstallationInfoForReview = DiffableRule & {
+  rule_id: RuleSignatureId;
+  version: RuleVersion;
+};

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/review_rule_upgrade/response_schema.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { RuleObjectId, RuleSignatureId, RuleTagArray } from '../../../rule_schema';
+import type { DiffableRule } from '../../model/diff/diffable_rule/diffable_rule';
+import type { PartialRuleDiff } from '../../model/diff/rule_diff/rule_diff';
+
+export interface ReviewRuleUpgradeResponseBody {
+  status_code: number;
+  message: string;
+  attributes: {
+    /** Aggregated info about all rules available for upgrade */
+    stats: RuleUpgradeStatsForReview;
+
+    /** Info about individual rules: one object per each rule available for upgrade */
+    rules: RuleUpgradeInfoForReview[];
+  };
+}
+
+export interface RuleUpgradeStatsForReview {
+  /** Number of installed prebuilt rules available for upgrade (stock + customized) */
+  num_rules_to_upgrade_total: number;
+
+  /** Number of installed prebuilt rules available for upgrade which are stock (non-customized) */
+  num_rules_to_upgrade_not_customized: number;
+
+  /** Number of installed prebuilt rules available for upgrade which are customized by the user */
+  num_rules_to_upgrade_customized: number;
+
+  /** A union of all tags of all rules available for upgrade */
+  tags: RuleTagArray;
+
+  /** A union of all fields "to be upgraded" across all the rules available for upgrade. An array of field names. */
+  fields: string[];
+}
+
+export interface RuleUpgradeInfoForReview {
+  id: RuleObjectId;
+  rule_id: RuleSignatureId;
+  rule: DiffableRule;
+  diff: PartialRuleDiff;
+}

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/api/urls.ts

@@ -5,7 +5,22 @@
  * 2.0.
  */
 
-import { DETECTION_ENGINE_RULES_URL as RULES } from '../../../constants';
+import {
+  DETECTION_ENGINE_RULES_URL as RULES,
+  INTERNAL_DETECTION_ENGINE_URL as INTERNAL,
+} from '../../../constants';
 
-export const PREBUILT_RULES_URL = `${RULES}/prepackaged` as const;
-export const PREBUILT_RULES_STATUS_URL = `${RULES}/prepackaged/_status` as const;
+const OLD_BASE_URL = `${RULES}/prepackaged` as const;
+const NEW_BASE_URL = `${INTERNAL}/prebuilt_rules` as const;
+
+export const PREBUILT_RULES_URL = OLD_BASE_URL;
+export const PREBUILT_RULES_STATUS_URL = `${OLD_BASE_URL}/_status` as const;
+
+export const GET_PREBUILT_RULES_STATUS_URL = `${NEW_BASE_URL}/status` as const;
+export const REVIEW_RULE_UPGRADE_URL = `${NEW_BASE_URL}/upgrade/_review` as const;
+export const PERFORM_RULE_UPGRADE_URL = `${NEW_BASE_URL}/upgrade/_perform` as const;
+export const REVIEW_RULE_INSTALLATION_URL = `${NEW_BASE_URL}/installation/_review` as const;
+export const PERFORM_RULE_INSTALLATION_URL = `${NEW_BASE_URL}/installation/_perform` as const;
+
+// Helper endpoints for development and testing. Should be removed later.
+export const GENERATE_ASSETS_URL = `${NEW_BASE_URL}/_generate_assets` as const;

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/index.ts

@@ -8,5 +8,3 @@
 export * from './api/get_prebuilt_rules_and_timelines_status/response_schema';
 export * from './api/install_prebuilt_rules_and_timelines/response_schema';
 export * from './api/urls';
-
-export * from './model/prebuilt_rule';

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/build_schema.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+import { orUndefined } from '../../../../rule_schema/model/build_rule_schemas';
+
+interface RuleFields<TRequired extends t.Props, TOptional extends t.Props> {
+  required: TRequired;
+  optional: TOptional;
+}
+
+export const buildSchema = <TRequired extends t.Props, TOptional extends t.Props>(
+  fields: RuleFields<TRequired, TOptional>
+) => {
+  return t.intersection([
+    t.exact(t.type(fields.required)),
+    t.exact(t.type(orUndefined(fields.optional))),
+  ]);
+};

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types.ts

@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+import { TimeDuration } from '@kbn/securitysolution-io-ts-types';
+import {
+  BuildingBlockType,
+  DataViewId,
+  IndexPatternArray,
+  KqlQueryLanguage,
+  RuleFilterArray,
+  RuleNameOverride as RuleNameOverrideFieldName,
+  RuleQuery,
+  TimelineTemplateId,
+  TimelineTemplateTitle,
+  TimestampOverride as TimestampOverrideFieldName,
+  TimestampOverrideFallbackDisabled,
+} from '../../../../rule_schema';
+import { saved_id } from '../../../../schemas/common';
+
+// -------------------------------------------------------------------------------------------------
+// Rule data source
+
+export enum DataSourceType {
+  'index_patterns' = 'index_patterns',
+  'data_view' = 'data_view',
+}
+
+export type DataSourceIndexPatterns = t.TypeOf<typeof DataSourceIndexPatterns>;
+export const DataSourceIndexPatterns = t.exact(
+  t.type({
+    type: t.literal(DataSourceType.index_patterns),
+    index_patterns: IndexPatternArray,
+  })
+);
+
+export type DataSourceDataView = t.TypeOf<typeof DataSourceDataView>;
+export const DataSourceDataView = t.exact(
+  t.type({
+    type: t.literal(DataSourceType.data_view),
+    data_view_id: DataViewId,
+  })
+);
+
+export type RuleDataSource = t.TypeOf<typeof RuleDataSource>;
+export const RuleDataSource = t.union([DataSourceIndexPatterns, DataSourceDataView]);
+
+// -------------------------------------------------------------------------------------------------
+// Rule data query
+
+export enum KqlQueryType {
+  'inline_query' = 'inline_query',
+  'saved_query' = 'saved_query',
+}
+
+export type InlineKqlQuery = t.TypeOf<typeof InlineKqlQuery>;
+export const InlineKqlQuery = t.exact(
+  t.type({
+    type: t.literal(KqlQueryType.inline_query),
+    query: RuleQuery,
+    language: KqlQueryLanguage,
+    filters: RuleFilterArray,
+  })
+);
+
+export type SavedKqlQuery = t.TypeOf<typeof SavedKqlQuery>;
+export const SavedKqlQuery = t.exact(
+  t.type({
+    type: t.literal(KqlQueryType.saved_query),
+    saved_query_id: saved_id,
+  })
+);
+
+export type RuleKqlQuery = t.TypeOf<typeof RuleKqlQuery>;
+export const RuleKqlQuery = t.union([InlineKqlQuery, SavedKqlQuery]);
+
+export type RuleEqlQuery = t.TypeOf<typeof RuleEqlQuery>;
+export const RuleEqlQuery = t.exact(
+  t.type({
+    query: RuleQuery,
+    language: t.literal('eql'),
+    filters: RuleFilterArray,
+  })
+);
+
+// -------------------------------------------------------------------------------------------------
+// Rule schedule
+
+export type RuleSchedule = t.TypeOf<typeof RuleSchedule>;
+export const RuleSchedule = t.exact(
+  t.type({
+    interval: TimeDuration({ allowedUnits: ['s', 'm', 'h'] }),
+    lookback: TimeDuration({ allowedUnits: ['s', 'm', 'h'] }),
+  })
+);
+
+// -------------------------------------------------------------------------------------------------
+// Rule name override
+
+export type RuleNameOverrideObject = t.TypeOf<typeof RuleNameOverrideObject>;
+export const RuleNameOverrideObject = t.exact(
+  t.type({
+    field_name: RuleNameOverrideFieldName,
+  })
+);
+
+// -------------------------------------------------------------------------------------------------
+// Timestamp override
+
+export type TimestampOverrideObject = t.TypeOf<typeof TimestampOverrideObject>;
+export const TimestampOverrideObject = t.exact(
+  t.type({
+    field_name: TimestampOverrideFieldName,
+    fallback_disabled: TimestampOverrideFallbackDisabled,
+  })
+);
+
+// -------------------------------------------------------------------------------------------------
+// Reference to a timeline template
+
+export type TimelineTemplateReference = t.TypeOf<typeof TimelineTemplateReference>;
+export const TimelineTemplateReference = t.exact(
+  t.type({
+    timeline_id: TimelineTemplateId,
+    timeline_title: TimelineTemplateTitle,
+  })
+);
+
+// -------------------------------------------------------------------------------------------------
+// Building block
+
+export type BuildingBlockObject = t.TypeOf<typeof BuildingBlockObject>;
+export const BuildingBlockObject = t.exact(
+  t.type({
+    type: BuildingBlockType,
+  })
+);