[Security Solution][Entity Analytics] APIs for Entity Store engine (e…

…lastic#191986)

This PR introduces the following API routes for setting up Entity Store
"engines":

<meta charset="utf-8"><b style="font-weight:normal;"
id="docs-internal-guid-9410c5d7-7fff-e873-6830-887939a306fb"><div
dir="ltr" style="margin-left:-0.75pt;" align="left">
Initialise Engine | POST /api/entity_store/engines/<entity_type>/init
-- | --
Start Engine | POST /api/entity_store/engines/<entity_type>/start
Stop Engine | POST /api/entity_store/engines/<entity_type>/stop
Delete Engine | DELETE /api/entity_store/engines/<entity_type>
Get engine | GET  /api/entity_store/engines/<entity_type>
List Engines | GET /api/entity_store/engines

</div></b>

The PR includes the following:
- Adding the `EntityManager` plugin (see elastic/obs-entities) as a
dependency of the Security Solution
 - The OpenAPI schemas for the new routes
 - The actual Kibana side endpoints
 - A `Saved Object` to track the installed engines
 - A new `EntityStoreDataClient`
 - A new feature flag `entityStoreEngineRoutesEnabled` 
 

### How to test

1. Add some host/user data
* Easiest is to use
[elastic/security-data-generator](https://github.com/elastic/security-documents-generator)
2. Make sure to add `entityStoreEngineRoutesEnabled` under
`xpack.securitySolution.enableExperimental` in your `kibana.dev.yml`
3. In kibana dev tools or your terminal, call the `INIT` route for
either `user` or `host`.
4. You should now see 2 transforms in kibana. Make sure to re-trigger
them if needed so they process the documents.
5. Check that new entities have been observed by querying the new
entities index via:
    *  `GET .entities.v1.latest.ea*/_search`
6. Check the other endpoints are working (`START`, `STOP`, `LIST`, etc)
7. Calling `DELETE` should remove the transforms



Implements elastic/security-team#10230

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -312,6 +312,12 @@
   "entity-discovery-api-key": [
     "apiKey"
   ],
+  "entity-engine-status": [
+    "filter",
+    "indexPattern",
+    "status",
+    "type"
+  ],
   "epm-packages": [
     "additional_spaces_installed_kibana",
     "es_index_patterns",

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1057,6 +1057,23 @@
       }
     }
   },
+  "entity-engine-status": {
+    "dynamic": false,
+    "properties": {
+      "filter": {
+        "type": "keyword"
+      },
+      "indexPattern": {
+        "type": "keyword"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "type": {
+        "type": "keyword"
+      }
+    }
+  },
   "epm-packages": {
     "properties": {
       "additional_spaces_installed_kibana": {

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -93,6 +93,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
         "entity-definition": "61be3e95966045122b55e181bb39658b1dc9bbe9",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
+        "entity-engine-status": "0738aa1a06d3361911740f8f166071ea43a00927",
         "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
         "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",

src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts

@@ -124,6 +124,7 @@ const previouslyRegisteredTypes = [
   'security-rule',
   'security-solution-signals-migration',
   'risk-engine-configuration',
+  'entity-engine-status',
   'server',
   'siem-detection-engine-rule-actions',
   'siem-detection-engine-rule-execution-info',

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Entity Store Common Schema
+ *   version: 1
+ */
+
+import { z } from '@kbn/zod';
+
+export type EntityType = z.infer<typeof EntityType>;
+export const EntityType = z.enum(['user', 'host']);
+export type EntityTypeEnum = typeof EntityType.enum;
+export const EntityTypeEnum = EntityType.enum;
+
+export type IndexPattern = z.infer<typeof IndexPattern>;
+export const IndexPattern = z.string();
+
+export type EngineStatus = z.infer<typeof EngineStatus>;
+export const EngineStatus = z.enum(['installing', 'started', 'stopped']);
+export type EngineStatusEnum = typeof EngineStatus.enum;
+export const EngineStatusEnum = EngineStatus.enum;
+
+export type EngineDescriptor = z.infer<typeof EngineDescriptor>;
+export const EngineDescriptor = z.object({
+  type: EntityType.optional(),
+  indexPattern: IndexPattern.optional(),
+  status: EngineStatus.optional(),
+  filter: z.string().optional(),
+});

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml

@@ -0,0 +1,37 @@
+openapi: 3.0.0
+info:
+  title: Entity Store Common Schema
+  description: Common schema for Entity Store
+  version: '1'
+paths: {}
+components:
+  schemas:
+
+    EntityType:
+      type: string
+      enum:
+        - user
+        - host
+
+    EngineDescriptor:
+      type: object
+      properties:
+        type:
+          $ref: '#/components/schemas/EntityType'
+        indexPattern:
+          $ref: '#/components/schemas/IndexPattern'
+        status:
+          $ref: '#/components/schemas/EngineStatus'
+        filter:
+          type: string
+
+    EngineStatus:
+      type: string
+      enum:
+        - installing
+        - started
+        - stopped
+
+    IndexPattern:
+      type: string
+

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/delete.gen.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Delete the entity store engine
+ *   version: 2023-10-31
+ */
+
+import { z } from '@kbn/zod';
+import { BooleanFromString } from '@kbn/zod-helpers';
+
+import { EntityType } from '../common.gen';
+
+export type DeleteEntityStoreRequestQuery = z.infer<typeof DeleteEntityStoreRequestQuery>;
+export const DeleteEntityStoreRequestQuery = z.object({
+  /**
+   * Control flag to also delete the entity data.
+   */
+  data: BooleanFromString.optional(),
+});
+export type DeleteEntityStoreRequestQueryInput = z.input<typeof DeleteEntityStoreRequestQuery>;
+
+export type DeleteEntityStoreRequestParams = z.infer<typeof DeleteEntityStoreRequestParams>;
+export const DeleteEntityStoreRequestParams = z.object({
+  /**
+   * The entity type of the store (either 'user' or 'host').
+   */
+  entityType: EntityType,
+});
+export type DeleteEntityStoreRequestParamsInput = z.input<typeof DeleteEntityStoreRequestParams>;
+
+export type DeleteEntityStoreResponse = z.infer<typeof DeleteEntityStoreResponse>;
+export const DeleteEntityStoreResponse = z.object({
+  deleted: z.boolean().optional(),
+});

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/delete.schema.yaml

@@ -0,0 +1,37 @@
+openapi: 3.0.0
+
+info:
+  title: Delete the entity store engine
+  version: '2023-10-31'
+paths:
+  /api/entity_store/engines/{entityType}:
+    delete:
+      x-labels: [ess, serverless]
+      x-codegen-enabled: true
+      operationId: DeleteEntityStore
+      summary: Delete the Entity Store engine
+      parameters:
+        - name: entityType
+          in: path
+          required: true
+          schema:
+            $ref: '../common.schema.yaml#/components/schemas/EntityType'
+          description: The entity type of the store (either 'user' or 'host').
+
+        - name: data
+          in: query
+          required: false
+          schema:
+            type: boolean
+          description: Control flag to also delete the entity data.
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:      
+                  deleted:
+                    type: boolean
+

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/get.gen.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get Entity Store engine
+ *   version: 2023-10-31
+ */
+
+import { z } from '@kbn/zod';
+
+import { EntityType, EngineDescriptor } from '../common.gen';
+
+export type GetEntityStoreEngineRequestParams = z.infer<typeof GetEntityStoreEngineRequestParams>;
+export const GetEntityStoreEngineRequestParams = z.object({
+  /**
+   * The entity type of the store (either 'user' or 'host').
+   */
+  entityType: EntityType,
+});
+export type GetEntityStoreEngineRequestParamsInput = z.input<
+  typeof GetEntityStoreEngineRequestParams
+>;
+
+export type GetEntityStoreEngineResponse = z.infer<typeof GetEntityStoreEngineResponse>;
+export const GetEntityStoreEngineResponse = EngineDescriptor;

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/get.schema.yaml

@@ -0,0 +1,25 @@
+openapi: 3.0.0
+info:
+  title: Get Entity Store engine 
+  version: '2023-10-31'
+paths:
+  /api/entity_store/engines/{entityType}:
+    get:
+      x-labels: [ess, serverless]
+      x-codegen-enabled: true
+      operationId: GetEntityStoreEngine
+      summary: Get the Entity Store engine
+      parameters:
+        - name: entityType
+          in: path
+          required: true
+          schema:
+            $ref: '../common.schema.yaml#/components/schemas/EntityType'
+          description: The entity type of the store (either 'user' or 'host').
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                $ref: '../common.schema.yaml#/components/schemas/EngineDescriptor'

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.gen.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Init Entity Store types
+ *   version: 2023-10-31
+ */
+
+import { z } from '@kbn/zod';
+
+import { EntityType, IndexPattern, EngineDescriptor } from '../common.gen';
+
+export type InitEntityStoreRequestParams = z.infer<typeof InitEntityStoreRequestParams>;
+export const InitEntityStoreRequestParams = z.object({
+  /**
+   * The entity type of the store (either 'user' or 'host').
+   */
+  entityType: EntityType,
+});
+export type InitEntityStoreRequestParamsInput = z.input<typeof InitEntityStoreRequestParams>;
+
+export type InitEntityStoreRequestBody = z.infer<typeof InitEntityStoreRequestBody>;
+export const InitEntityStoreRequestBody = z.object({
+  indexPattern: IndexPattern.optional(),
+  filter: z.string().optional(),
+});
+export type InitEntityStoreRequestBodyInput = z.input<typeof InitEntityStoreRequestBody>;
+
+export type InitEntityStoreResponse = z.infer<typeof InitEntityStoreResponse>;
+export const InitEntityStoreResponse = EngineDescriptor;

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml

@@ -0,0 +1,39 @@
+openapi: 3.0.0
+
+info:
+  title: Init Entity Store types
+  version: '2023-10-31'
+paths:
+   /api/entity_store/engines/{entityType}/init:
+    post:
+      x-labels: [ess, serverless]
+      x-codegen-enabled: true
+      operationId: InitEntityStore
+      summary: Initialize the Entity Store
+      parameters:
+        - name: entityType
+          in: path
+          required: true
+          schema:
+            $ref: '../common.schema.yaml#/components/schemas/EntityType'
+          description: The entity type of the store (either 'user' or 'host').
+      requestBody:
+        description: Schema for the engine initialization
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                indexPattern:
+                  $ref: '../common.schema.yaml#/components/schemas/IndexPattern'
+                filter:
+                  type: string
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                $ref: '../common.schema.yaml#/components/schemas/EngineDescriptor'
+

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/list.gen.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: List Entity Store engines
+ *   version: 2023-10-31
+ */
+
+import { z } from '@kbn/zod';
+
+import { EngineDescriptor } from '../common.gen';
+
+export type ListEntityStoreEnginesResponse = z.infer<typeof ListEntityStoreEnginesResponse>;
+export const ListEntityStoreEnginesResponse = z.object({
+  count: z.number().int().optional(),
+  engines: z.array(EngineDescriptor).optional(),
+});