From 9b68804e0cd6f2ea04083b0627b9fc871f455f1f Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Wed, 12 May 2021 16:57:15 -0400 Subject: [PATCH] [Security Solution][Endpoint] Fix generator so that the `--fleet` option creates a fake agent with fleet (#99942) * new fleet agent generator * Indexing of generated data changed to index fake fleet server agents --- .../data_generators/fleet_agent_generator.ts | 116 +++++++ .../common/endpoint/index_data.ts | 293 ++++++------------ 2 files changed, 211 insertions(+), 198 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_agent_generator.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_agent_generator.ts b/x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_agent_generator.ts new file mode 100644 index 0000000000000..6e508a099003a --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_agent_generator.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { DeepPartial } from 'utility-types'; +import { merge } from 'lodash'; +import { BaseDataGenerator } from './base_data_generator'; +import { Agent, AGENTS_INDEX, FleetServerAgent } from '../../../../fleet/common'; + +export class FleetAgentGenerator extends BaseDataGenerator { + /** + * @param [overrides] any partial value to the full Agent record + * + * @example + * + * fleetAgentGenerator.generate({ + * local_metadata: { + * elastic: { + * agent: { + * log_level: `debug` + * } + * } + * } + * }); + */ + generate(overrides: DeepPartial = {}): Agent { + const hit = this.generateEsHit(); + + // The mapping below is identical to `searchHitToAgent()` located in + // `x-pack/plugins/fleet/server/services/agents/helpers.ts:19` + return merge( + { + // Casting here is needed because several of the attributes in `FleetServerAgent` are + // defined as optional, but required in `Agent` type. + ...(hit._source as Agent), + id: hit._id, + policy_revision: hit._source?.policy_revision_idx, + access_api_key: undefined, + status: undefined, + packages: hit._source?.packages ?? [], + }, + overrides + ); + } + + /** + * @param [overrides] any partial value to the full document + */ + generateEsHit( + overrides: DeepPartial> = {} + ): estypes.Hit { + const hostname = this.randomHostname(); + const now = new Date().toISOString(); + const osFamily = this.randomOSFamily(); + + return merge, DeepPartial>>( + { + _index: AGENTS_INDEX, + _id: this.randomUUID(), + _score: 1.0, + _source: { + access_api_key_id: 'jY3dWnkBj1tiuAw9pAmq', + action_seq_no: -1, + active: true, + enrolled_at: now, + local_metadata: { + elastic: { + agent: { + 'build.original': `8.0.0-SNAPSHOT (build: ${this.randomString( + 5 + )} at 2021-05-07 18:42:49 +0000 UTC)`, + id: this.randomUUID(), + log_level: 'info', + snapshot: true, + upgradeable: true, + version: '8.0.0', + }, + }, + host: { + architecture: 'x86_64', + hostname, + id: this.randomUUID(), + ip: [this.randomIP()], + mac: [this.randomMac()], + name: hostname, + }, + os: { + family: osFamily, + full: `${osFamily} 2019 Datacenter`, + kernel: '10.0.17763.1879 (Build.160101.0800)', + name: `${osFamily} Server 2019 Datacenter`, + platform: osFamily, + version: this.randomVersion(), + }, + }, + user_provided_metadata: {}, + policy_id: this.randomUUID(), + type: 'PERMANENT', + default_api_key: 'so3dWnkBj1tiuAw9yAm3:t7jNlnPnR6azEI_YpXuBXQ', + // policy_output_permissions_hash: + // '81b3d070dddec145fafcbdfb6f22888495a12edc31881f6b0511fa10de66daa7', + default_api_key_id: 'so3dWnkBj1tiuAw9yAm3', + updated_at: now, + last_checkin: now, + policy_revision_idx: 2, + policy_coordinator_idx: 1, + }, + }, + overrides + ); + } +} diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index fd26a2d95c9b4..0dc7891560c2d 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -5,31 +5,31 @@ * 2.0. */ -import { Client } from '@elastic/elasticsearch'; +import { Client, estypes } from '@elastic/elasticsearch'; import seedrandom from 'seedrandom'; // eslint-disable-next-line import/no-extraneous-dependencies import { KbnClient } from '@kbn/test'; import { AxiosResponse } from 'axios'; -import { EndpointDocGenerator, TreeOptions, Event } from './generate_data'; +import { EndpointDocGenerator, Event, TreeOptions } from './generate_data'; import { firstNonNullValue } from './models/ecs_safety_helpers'; import { + AGENT_POLICY_API_ROUTES, CreateAgentPolicyRequest, CreateAgentPolicyResponse, CreatePackagePolicyRequest, CreatePackagePolicyResponse, - GetPackagesResponse, - AGENT_API_ROUTES, - AGENT_POLICY_API_ROUTES, EPM_API_ROUTES, + FLEET_SERVER_SERVERS_INDEX, + FleetServerAgent, + GetPackagesResponse, PACKAGE_POLICY_API_ROUTES, - ENROLLMENT_API_KEY_ROUTES, - GetEnrollmentAPIKeysResponse, - GetOneEnrollmentAPIKeyResponse, - Agent, } from '../../../fleet/common'; import { policyFactory as policyConfigFactory } from './models/policy_config'; import { HostMetadata } from './types'; import { KbnClientWithApiKeySupport } from '../../scripts/endpoint/kbn_client_with_api_key_support'; +import { FleetAgentGenerator } from './data_generators/fleet_agent_generator'; + +const fleetAgentGenerator = new FleetAgentGenerator(); export async function indexHostsAndAlerts( client: Client, @@ -47,8 +47,15 @@ export async function indexHostsAndAlerts( ) { const random = seedrandom(seed); const epmEndpointPackage = await getEndpointPackageInfo(kbnClient); + + // If `fleet` integration is true, then ensure a (fake) fleet-server is connected + if (fleet) { + await enableFleetServerIfNecessary(client); + } + // Keep a map of host applied policy ids (fake) to real ingest package configs (policy record) const realPolicies: Record = {}; + for (let i = 0; i < numHosts; i++) { const generator = new EndpointDocGenerator(random); await indexHostDocs({ @@ -71,9 +78,11 @@ export async function indexHostsAndAlerts( options, }); } + await client.indices.refresh({ index: eventIndex, }); + // TODO: Unclear why the documents are not showing up after the call to refresh. // Waiting 5 seconds allows the indices to refresh automatically and // the documents become available in API/integration tests. @@ -107,9 +116,10 @@ async function indexHostDocs({ }) { const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents const timestamp = new Date().getTime(); + const kibanaVersion = await fetchKibanaVersion(kbnClient); let hostMetadata: HostMetadata; let wasAgentEnrolled = false; - let enrolledAgent: undefined | Agent; + let enrolledAgent: undefined | estypes.Hit; for (let j = 0; j < numDocs; j++) { generator.updateHostData(); @@ -136,10 +146,12 @@ async function indexHostDocs({ // If we did not yet enroll an agent for this Host, do it now that we have good policy id if (!wasAgentEnrolled) { wasAgentEnrolled = true; - enrolledAgent = await fleetEnrollAgentForHost( - kbnClient, + + enrolledAgent = await indexFleetAgentForHost( + client, hostMetadata!, - realPolicies[appliedPolicyId].policy_id + realPolicies[appliedPolicyId].policy_id, + kibanaVersion ); } // Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id @@ -149,7 +161,7 @@ async function indexHostDocs({ ...hostMetadata.elastic, agent: { ...hostMetadata.elastic.agent, - id: enrolledAgent?.id ?? hostMetadata.elastic.agent.id, + id: enrolledAgent?._id ?? hostMetadata.elastic.agent.id, }, }, Endpoint: { @@ -295,208 +307,93 @@ const getEndpointPackageInfo = async ( return endpointPackage; }; -const fleetEnrollAgentForHost = async ( - kbnClient: KbnClientWithApiKeySupport, - endpointHost: HostMetadata, - agentPolicyId: string -): Promise => { - // Get Enrollement key for host's applied policy - const enrollmentApiKey = await kbnClient - .request({ - path: ENROLLMENT_API_KEY_ROUTES.LIST_PATTERN, - method: 'GET', - query: { - kuery: `policy_id:"${agentPolicyId}"`, - }, - }) - .then((apiKeysResponse) => { - const apiKey = apiKeysResponse.data.list[0]; +const fetchKibanaVersion = async (kbnClient: KbnClientWithApiKeySupport) => { + const version = ((await kbnClient.request({ + path: '/api/status', + method: 'GET', + })) as AxiosResponse).data.version.number; - if (!apiKey) { - return Promise.reject( - new Error(`no API enrollment key found for agent policy id ${agentPolicyId}`) - ); - } + if (!version) { + // eslint-disable-next-line no-console + console.log('failed to retrieve kibana version'); + return '8.0.0'; + } - return kbnClient - .request({ - path: ENROLLMENT_API_KEY_ROUTES.INFO_PATTERN.replace('{keyId}', apiKey.id), - method: 'GET', - }) - .catch((error) => { - // eslint-disable-next-line no-console - console.log('unable to retrieve enrollment api key for policy'); - return Promise.reject(error); - }); - }) - .then((apiKeyDetailsResponse) => { - return apiKeyDetailsResponse.data.item.api_key; - }) - .catch((error) => { - // eslint-disable-next-line no-console - console.error(error); - return ''; - }); + return version; +}; + +/** + * Will ensure that at least one fleet server is present in the `.fleet-servers` index. This will + * enable the `Agent` section of kibana Fleet to be displayed + * + * @param esClient + * @param version + */ +const enableFleetServerIfNecessary = async (esClient: Client, version: string = '8.0.0') => { + const res = await esClient.search<{}, {}>({ + index: FLEET_SERVER_SERVERS_INDEX, + ignore_unavailable: true, + }); - if (enrollmentApiKey.length === 0) { + // @ts-expect-error value is number | TotalHits + if (res.body.hits.total.value > 0) { return; } - const fetchKibanaVersion = async () => { - const version = ((await kbnClient.request({ - path: '/api/status', - method: 'GET', - })) as AxiosResponse).data.version.number; - if (!version) { - // eslint-disable-next-line no-console - console.log('failed to retrieve kibana version'); - } - return version; - }; + // Create a Fake fleet-server in this kibana instance + await esClient.index({ + index: FLEET_SERVER_SERVERS_INDEX, + body: { + agent: { + id: '12988155-475c-430d-ac89-84dc84b67cd1', + version: '', + }, + host: { + architecture: 'linux', + id: 'c3e5f4f690b4a3ff23e54900701a9513', + ip: ['127.0.0.1', '::1', '10.201.0.213', 'fe80::4001:aff:fec9:d5'], + name: 'endpoint-data-generator', + }, + server: { + id: '12988155-475c-430d-ac89-84dc84b67cd1', + version: '8.0.0-SNAPSHOT', + }, + '@timestamp': '2021-05-12T18:42:52.009482058Z', + }, + }); +}; - // Enroll an agent for the Host - const body = { - type: 'PERMANENT', - metadata: { - local: { +const indexFleetAgentForHost = async ( + esClient: Client, + endpointHost: HostMetadata, + agentPolicyId: string, + kibanaVersion: string = '8.0.0' +): Promise> => { + const agentDoc = fleetAgentGenerator.generateEsHit({ + _source: { + local_metadata: { elastic: { agent: { - version: await fetchKibanaVersion(), + version: kibanaVersion, }, }, host: { ...endpointHost.host, }, os: { - family: 'windows', - kernel: '10.0.19041.388 (WinBuild.160101.0800)', - platform: 'windows', - version: '10.0', - name: 'Windows 10 Pro', - full: 'Windows 10 Pro(10.0)', + ...endpointHost.host.os, }, }, - user_provided: { - dev_agent_version: '0.0.1', - region: 'us-east', - }, + policy_id: agentPolicyId, }, - }; - - try { - // First enroll the agent - const res = await kbnClient.requestWithApiKey(AGENT_API_ROUTES.ENROLL_PATTERN, { - method: 'POST', - body: JSON.stringify(body), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollmentApiKey}`, - 'Content-Type': 'application/json', - }, - }); - - if (res) { - const enrollObj = await res.json(); - if (!res.ok) { - // eslint-disable-next-line no-console - console.error('unable to enroll agent', enrollObj); - return; - } - // ------------------------------------------------ - // now check the agent in so that it can complete enrollment - const checkinBody = { - events: [ - { - type: 'STATE', - subtype: 'RUNNING', - message: 'state changed from STOPPED to RUNNING', - timestamp: new Date().toISOString(), - payload: { - random: 'data', - state: 'RUNNING', - previous_state: 'STOPPED', - }, - agent_id: enrollObj.item.id, - }, - ], - }; - const checkinRes = await kbnClient - .requestWithApiKey( - AGENT_API_ROUTES.CHECKIN_PATTERN.replace('{agentId}', enrollObj.item.id), - { - method: 'POST', - body: JSON.stringify(checkinBody), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollObj.item.access_api_key}`, - 'Content-Type': 'application/json', - }, - } - ) - .catch((error) => { - return Promise.reject(error); - }); - - // Agent unenrolling? - if (checkinRes.status === 403) { - return; - } - - const checkinObj = await checkinRes.json(); - if (!checkinRes.ok) { - // eslint-disable-next-line no-console - console.error( - `failed to checkin agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` - ); - return enrollObj.item; - } - - // ------------------------------------------------ - // If we have an action to ack(), then do it now - if (checkinObj.actions.length) { - const ackActionBody = { - // @ts-ignore - events: checkinObj.actions.map((action) => { - return { - action_id: action.id, - type: 'ACTION_RESULT', - subtype: 'CONFIG', - timestamp: new Date().toISOString(), - agent_id: action.agent_id, - policy_id: agentPolicyId, - message: `endpoint generator: Endpoint Started`, - }; - }), - }; - const ackActionResp = await kbnClient.requestWithApiKey( - AGENT_API_ROUTES.ACKS_PATTERN.replace('{agentId}', enrollObj.item.id), - { - method: 'POST', - body: JSON.stringify(ackActionBody), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollObj.item.access_api_key}`, - 'Content-Type': 'application/json', - }, - } - ); + }); - const ackActionObj = await ackActionResp.json(); - if (!ackActionResp.ok) { - // eslint-disable-next-line no-console - console.error( - `failed to ACK Actions provided to agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` - ); - // eslint-disable-next-line no-console - console.error(JSON.stringify(ackActionObj, null, 2)); - return enrollObj.item; - } - } + await esClient.index({ + index: agentDoc._index, + id: agentDoc._id, + body: agentDoc._source!, + op_type: 'create', + }); - return enrollObj.item; - } - } catch (error) { - // eslint-disable-next-line no-console - console.error(error); - } + return agentDoc; };