From 755ba3ab0816d4f66cf797bb013af83e38926d5a Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Thu, 12 Mar 2020 13:56:40 -0700 Subject: [PATCH] [Ingest] Add `revision` to agent configs & data sources (#59848) * Add revision to agent config and datasource saved objects, add delete datasource service and datasource * Add revision to full agent config output * PR feedback --- .../ingest_manager/common/constants/routes.ts | 1 + .../datasource_to_agent_datasource.ts | 3 - .../common/types/models/agent_config.ts | 4 +- .../common/types/models/datasource.ts | 5 +- .../sections/agent_config/list_page/index.tsx | 2 +- .../server/routes/datasource/handlers.ts | 40 ++++++++--- .../server/routes/datasource/index.ts | 12 ++++ .../ingest_manager/server/saved_objects.ts | 2 + .../server/services/agent_config.ts | 41 +++++++---- .../server/services/agent_config_update.ts | 6 +- .../server/services/agents/update.ts | 4 +- .../server/services/datasource.ts | 72 +++++++++++++++++-- .../server/types/models/agent_config.ts | 2 +- 13 files changed, 151 insertions(+), 43 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/constants/routes.ts b/x-pack/plugins/ingest_manager/common/constants/routes.ts index a08301c13622b..1dc98f9bc8947 100644 --- a/x-pack/plugins/ingest_manager/common/constants/routes.ts +++ b/x-pack/plugins/ingest_manager/common/constants/routes.ts @@ -28,6 +28,7 @@ export const DATASOURCE_API_ROUTES = { INFO_PATTERN: `${DATASOURCE_API_ROOT}/{datasourceId}`, CREATE_PATTERN: `${DATASOURCE_API_ROOT}`, UPDATE_PATTERN: `${DATASOURCE_API_ROOT}/{datasourceId}`, + DELETE_PATTERN: `${DATASOURCE_API_ROOT}/delete`, }; // Agent config API routes diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts index 93d6538283241..57627fa60fe43 100644 --- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts +++ b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts @@ -15,7 +15,6 @@ export const storedDatasourceToAgentDatasource = ( namespace, enabled, use_output: DEFAULT_OUTPUT.name, // TODO: hardcoded to default output for now - package: undefined, inputs: inputs .filter(input => input.enabled) .map(input => ({ @@ -46,8 +45,6 @@ export const storedDatasourceToAgentDatasource = ( name: pkg.name, version: pkg.version, }; - } else { - delete fullDatasource.package; } return fullDatasource; diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts index c78eb72892703..c63e496273ada 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts @@ -20,7 +20,7 @@ export enum AgentConfigStatus { export interface NewAgentConfig { name: string; - namespace: string; + namespace?: string; description?: string; is_default?: boolean; } @@ -31,6 +31,7 @@ export interface AgentConfig extends NewAgentConfig, SavedObjectAttributes { datasources: string[] | Datasource[]; updated_on: string; updated_by: string; + revision: number; } export type FullAgentConfigDatasource = Pick & { @@ -56,4 +57,5 @@ export interface FullAgentConfig { }; }; datasources: FullAgentConfigDatasource[]; + revision?: number; } diff --git a/x-pack/plugins/ingest_manager/common/types/models/datasource.ts b/x-pack/plugins/ingest_manager/common/types/models/datasource.ts index 9515b16007332..3503bbdcd40e3 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/datasource.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/datasource.ts @@ -36,4 +36,7 @@ export interface NewDatasource { inputs: DatasourceInput[]; } -export type Datasource = NewDatasource & { id: string }; +export type Datasource = NewDatasource & { + id: string; + revision: number; +}; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx index 49399be092a89..35915fab6f143 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx @@ -239,7 +239,7 @@ export const AgentConfigListPage: React.FunctionComponent<{}> = () => { diff --git a/x-pack/plugins/ingest_manager/server/routes/datasource/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/datasource/handlers.ts index 51a5f566d9f28..349e88d8fb59d 100644 --- a/x-pack/plugins/ingest_manager/server/routes/datasource/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/datasource/handlers.ts @@ -5,15 +5,16 @@ */ import { TypeOf } from '@kbn/config-schema'; import { RequestHandler } from 'kibana/server'; -import { appContextService, datasourceService, agentConfigService } from '../../services'; +import { appContextService, datasourceService } from '../../services'; import { ensureInstalledPackage } from '../../services/epm/packages'; import { GetDatasourcesRequestSchema, GetOneDatasourceRequestSchema, CreateDatasourceRequestSchema, UpdateDatasourceRequestSchema, + DeleteDatasourcesRequestSchema, } from '../../types'; -import { CreateDatasourceResponse } from '../../../common'; +import { CreateDatasourceResponse, DeleteDatasourcesResponse } from '../../../common'; export const getDatasourcesHandler: RequestHandler< undefined, @@ -85,12 +86,7 @@ export const createDatasourceHandler: RequestHandler< } // Create datasource - const datasource = await datasourceService.create(soClient, request.body); - - // Assign it to the given agent config - await agentConfigService.assignDatasources(soClient, datasource.config_id, [datasource.id], { - user, - }); + const datasource = await datasourceService.create(soClient, request.body, { user }); const body: CreateDatasourceResponse = { item: datasource, success: true }; return response.ok({ body, @@ -109,11 +105,13 @@ export const updateDatasourceHandler: RequestHandler< TypeOf > = async (context, request, response) => { const soClient = context.core.savedObjects.client; + const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined; try { const datasource = await datasourceService.update( soClient, request.params.datasourceId, - request.body + request.body, + { user } ); return response.ok({ body: { item: datasource, success: true }, @@ -125,3 +123,27 @@ export const updateDatasourceHandler: RequestHandler< }); } }; + +export const deleteDatasourceHandler: RequestHandler< + unknown, + unknown, + TypeOf +> = async (context, request, response) => { + const soClient = context.core.savedObjects.client; + const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined; + try { + const body: DeleteDatasourcesResponse = await datasourceService.delete( + soClient, + request.body.datasourceIds, + { user } + ); + return response.ok({ + body, + }); + } catch (e) { + return response.customError({ + statusCode: 500, + body: { message: e.message }, + }); + } +}; diff --git a/x-pack/plugins/ingest_manager/server/routes/datasource/index.ts b/x-pack/plugins/ingest_manager/server/routes/datasource/index.ts index 412eb17c6d45a..e5891cc7377e9 100644 --- a/x-pack/plugins/ingest_manager/server/routes/datasource/index.ts +++ b/x-pack/plugins/ingest_manager/server/routes/datasource/index.ts @@ -10,12 +10,14 @@ import { GetOneDatasourceRequestSchema, CreateDatasourceRequestSchema, UpdateDatasourceRequestSchema, + DeleteDatasourcesRequestSchema, } from '../../types'; import { getDatasourcesHandler, getOneDatasourceHandler, createDatasourceHandler, updateDatasourceHandler, + deleteDatasourceHandler, } from './handlers'; export const registerRoutes = (router: IRouter) => { @@ -58,4 +60,14 @@ export const registerRoutes = (router: IRouter) => { }, updateDatasourceHandler ); + + // Delete + router.post( + { + path: DATASOURCE_API_ROUTES.DELETE_PATTERN, + validate: DeleteDatasourcesRequestSchema, + options: { tags: [`access:${PLUGIN_ID}`] }, + }, + deleteDatasourceHandler + ); }; diff --git a/x-pack/plugins/ingest_manager/server/saved_objects.ts b/x-pack/plugins/ingest_manager/server/saved_objects.ts index 8d9da60065e7a..860b95b58c7f7 100644 --- a/x-pack/plugins/ingest_manager/server/saved_objects.ts +++ b/x-pack/plugins/ingest_manager/server/saved_objects.ts @@ -75,6 +75,7 @@ export const savedObjectMappings = { datasources: { type: 'keyword' }, updated_on: { type: 'keyword' }, updated_by: { type: 'keyword' }, + revision: { type: 'integer' }, }, }, [ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE]: { @@ -138,6 +139,7 @@ export const savedObjectMappings = { }, }, }, + revision: { type: 'integer' }, }, }, [PACKAGES_SAVED_OBJECT_TYPE]: { diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.ts index 75a016fc18e69..c0eb614102b29 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.ts @@ -36,8 +36,24 @@ class AgentConfigService { agentConfig: Partial, user?: AuthenticatedUser ): Promise { + const oldAgentConfig = await this.get(soClient, id, false); + + if (!oldAgentConfig) { + throw new Error('Agent config not found'); + } + + if ( + oldAgentConfig.status === AgentConfigStatus.Inactive && + agentConfig.status !== AgentConfigStatus.Active + ) { + throw new Error( + `Agent config ${id} cannot be updated because it is ${oldAgentConfig.status}` + ); + } + await soClient.update(SAVED_OBJECT_TYPE, id, { ...agentConfig, + revision: oldAgentConfig.revision + 1, updated_on: new Date().toString(), updated_by: user ? user.username : 'system', }); @@ -76,6 +92,7 @@ class AgentConfigService { SAVED_OBJECT_TYPE, { ...agentConfig, + revision: 1, updated_on: new Date().toISOString(), updated_by: options?.user?.username || 'system', } as AgentConfig, @@ -160,24 +177,17 @@ class AgentConfigService { agentConfig: Partial, options?: { user?: AuthenticatedUser } ): Promise { - const oldAgentConfig = await this.get(soClient, id); - - if (!oldAgentConfig) { - throw new Error('Agent config not found'); - } - - if ( - oldAgentConfig.status === AgentConfigStatus.Inactive && - agentConfig.status !== AgentConfigStatus.Active - ) { - throw new Error( - `Agent config ${id} cannot be updated because it is ${oldAgentConfig.status}` - ); - } - return this._update(soClient, id, agentConfig, options?.user); } + public async bumpRevision( + soClient: SavedObjectsClientContract, + id: string, + options?: { user?: AuthenticatedUser } + ): Promise { + return this._update(soClient, id, {}, options?.user); + } + public async assignDatasources( soClient: SavedObjectsClientContract, id: string, @@ -307,6 +317,7 @@ class AgentConfigService { datasources: (config.datasources as Datasource[]).map(ds => storedDatasourceToAgentDatasource(ds) ), + revision: config.revision, }; return agentConfig; diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config_update.ts b/x-pack/plugins/ingest_manager/server/services/agent_config_update.ts index c0a40ffbe034a..38894ff321a0b 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config_update.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config_update.ts @@ -6,7 +6,7 @@ import { SavedObjectsClientContract } from 'kibana/server'; import { generateEnrollmentAPIKey, deleteEnrollmentApiKeyForConfigId } from './api_keys'; -import { updateAgentsForPolicyId, unenrollForPolicyId } from './agents'; +import { updateAgentsForConfigId, unenrollForConfigId } from './agents'; export async function agentConfigUpdateEventHandler( soClient: SavedObjectsClientContract, @@ -20,11 +20,11 @@ export async function agentConfigUpdateEventHandler( } if (action === 'updated') { - await updateAgentsForPolicyId(soClient, configId); + await updateAgentsForConfigId(soClient, configId); } if (action === 'deleted') { - await unenrollForPolicyId(soClient, configId); + await unenrollForConfigId(soClient, configId); await deleteEnrollmentApiKeyForConfigId(soClient, configId); } } diff --git a/x-pack/plugins/ingest_manager/server/services/agents/update.ts b/x-pack/plugins/ingest_manager/server/services/agents/update.ts index 152a51ae519bf..8452c05d53a1f 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/update.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/update.ts @@ -9,7 +9,7 @@ import { listAgents } from './crud'; import { AGENT_SAVED_OBJECT_TYPE } from '../../constants'; import { unenrollAgents } from './unenroll'; -export async function updateAgentsForPolicyId( +export async function updateAgentsForConfigId( soClient: SavedObjectsClientContract, configId: string ) { @@ -37,7 +37,7 @@ export async function updateAgentsForPolicyId( } } -export async function unenrollForPolicyId(soClient: SavedObjectsClientContract, configId: string) { +export async function unenrollForConfigId(soClient: SavedObjectsClientContract, configId: string) { let hasMore = true; let page = 1; while (hasMore) { diff --git a/x-pack/plugins/ingest_manager/server/services/datasource.ts b/x-pack/plugins/ingest_manager/server/services/datasource.ts index ada1a999b648f..615b29087ba1e 100644 --- a/x-pack/plugins/ingest_manager/server/services/datasource.ts +++ b/x-pack/plugins/ingest_manager/server/services/datasource.ts @@ -4,8 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ import { SavedObjectsClientContract } from 'kibana/server'; +import { AuthenticatedUser } from '../../../security/server'; +import { DeleteDatasourcesResponse } from '../../common'; import { DATASOURCE_SAVED_OBJECT_TYPE } from '../constants'; import { NewDatasource, Datasource, ListWithKuery } from '../types'; +import { agentConfigService } from './agent_config'; const SAVED_OBJECT_TYPE = DATASOURCE_SAVED_OBJECT_TYPE; @@ -13,14 +16,22 @@ class DatasourceService { public async create( soClient: SavedObjectsClientContract, datasource: NewDatasource, - options?: { id?: string } + options?: { id?: string; user?: AuthenticatedUser } ): Promise { - const newSo = await soClient.create( + const newSo = await soClient.create>( SAVED_OBJECT_TYPE, - datasource as Datasource, + { + ...datasource, + revision: 1, + }, options ); + // Assign it to the given agent config + await agentConfigService.assignDatasources(soClient, datasource.config_id, [newSo.id], { + user: options?.user, + }); + return { id: newSo.id, ...newSo.attributes, @@ -98,14 +109,61 @@ class DatasourceService { public async update( soClient: SavedObjectsClientContract, id: string, - datasource: NewDatasource + datasource: NewDatasource, + options?: { user?: AuthenticatedUser } ): Promise { - await soClient.update(SAVED_OBJECT_TYPE, id, datasource); + const oldDatasource = await this.get(soClient, id); + + if (!oldDatasource) { + throw new Error('Datasource not found'); + } + + await soClient.update(SAVED_OBJECT_TYPE, id, { + ...datasource, + revision: oldDatasource.revision + 1, + }); + + // Bump revision of associated agent config + await agentConfigService.bumpRevision(soClient, datasource.config_id, { user: options?.user }); + return (await this.get(soClient, id)) as Datasource; } - public async delete(soClient: SavedObjectsClientContract, id: string): Promise { - await soClient.delete(SAVED_OBJECT_TYPE, id); + public async delete( + soClient: SavedObjectsClientContract, + ids: string[], + options?: { user?: AuthenticatedUser } + ): Promise { + const result: DeleteDatasourcesResponse = []; + + for (const id of ids) { + try { + const oldDatasource = await this.get(soClient, id); + if (!oldDatasource) { + throw new Error('Datasource not found'); + } + await agentConfigService.unassignDatasources( + soClient, + oldDatasource.config_id, + [oldDatasource.id], + { + user: options?.user, + } + ); + await soClient.delete(SAVED_OBJECT_TYPE, id); + result.push({ + id, + success: true, + }); + } catch (e) { + result.push({ + id, + success: false, + }); + } + } + + return result; } } diff --git a/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts index 0179173871bd2..040b2eb16289a 100644 --- a/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts @@ -9,7 +9,7 @@ import { AgentConfigStatus } from '../../../common'; const AgentConfigBaseSchema = { name: schema.string(), - namespace: schema.string(), + namespace: schema.maybe(schema.string()), description: schema.maybe(schema.string()), };