From 76df74d981ab53ed421f8826633ca04251bace8e Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Thu, 17 Sep 2020 16:43:46 -0400 Subject: [PATCH 01/10] Adding bulk upgrade api --- .../ingest_manager/common/constants/routes.ts | 2 + .../common/types/rest_spec/epm.ts | 20 +++ .../ingest_manager/server/errors/handlers.ts | 28 ++-- .../server/routes/epm/handlers.ts | 59 ++++---- .../ingest_manager/server/routes/epm/index.ts | 11 ++ .../server/services/epm/packages/install.ts | 119 +++++++++++++++- .../server/types/rest_spec/epm.ts | 6 + .../apis/epm/bulk_upgrade.ts | 130 ++++++++++++++++++ .../apis/epm/index.js | 1 + 9 files changed, 333 insertions(+), 43 deletions(-) create mode 100644 x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts diff --git a/x-pack/plugins/ingest_manager/common/constants/routes.ts b/x-pack/plugins/ingest_manager/common/constants/routes.ts index 3e065142ea101..41b877f476bfc 100644 --- a/x-pack/plugins/ingest_manager/common/constants/routes.ts +++ b/x-pack/plugins/ingest_manager/common/constants/routes.ts @@ -14,10 +14,12 @@ export const FLEET_API_ROOT = `${API_ROOT}/fleet`; export const LIMITED_CONCURRENCY_ROUTE_TAG = 'ingest:limited-concurrency'; // EPM API routes +const EPM_PACKAGES_BULK = `${EPM_API_ROOT}/packages_bulk`; const EPM_PACKAGES_MANY = `${EPM_API_ROOT}/packages`; const EPM_PACKAGES_ONE = `${EPM_PACKAGES_MANY}/{pkgkey}`; const EPM_PACKAGES_FILE = `${EPM_PACKAGES_MANY}/{pkgName}/{pkgVersion}`; export const EPM_API_ROUTES = { + BULK_UPGRADE_PATTERN: EPM_PACKAGES_BULK, LIST_PATTERN: EPM_PACKAGES_MANY, LIMITED_LIST_PATTERN: `${EPM_PACKAGES_MANY}/limited`, INFO_PATTERN: EPM_PACKAGES_ONE, diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts index 54e767fee4b22..d3459dacad596 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts @@ -71,6 +71,26 @@ export interface InstallPackageResponse { response: AssetReference[]; } +export interface UpgradePackageError { + name: string; + statusCode: number; + body: { + message: string; + }; +} + +export interface UpgradePackageInfo { + name: string; + newVersion: string; + // this will be null if no package was present before the upgrade (aka it was an install) + oldVersion: string | null; + assets: AssetReference[]; +} + +export interface UpgradePackagesResponse { + response: Array; +} + export interface MessageResponse { response: string; } diff --git a/x-pack/plugins/ingest_manager/server/errors/handlers.ts b/x-pack/plugins/ingest_manager/server/errors/handlers.ts index 9f776565cf262..8998674754dde 100644 --- a/x-pack/plugins/ingest_manager/server/errors/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/errors/handlers.ts @@ -56,10 +56,7 @@ const getHTTPResponseCode = (error: IngestManagerError): number => { return 400; // Bad Request }; -export const defaultIngestErrorHandler: IngestErrorHandler = async ({ - error, - response, -}: IngestErrorHandlerParams): Promise => { +export function handleIngestError(error: IngestManagerError | Boom | Error) { const logger = appContextService.getLogger(); if (isLegacyESClientError(error)) { // there was a problem communicating with ES (e.g. via `callCluster`) @@ -72,36 +69,43 @@ export const defaultIngestErrorHandler: IngestErrorHandler = async ({ logger.error(message); - return response.customError({ + return { statusCode: error?.statusCode || error.status, body: { message }, - }); + }; } // our "expected" errors if (error instanceof IngestManagerError) { // only log the message logger.error(error.message); - return response.customError({ + return { statusCode: getHTTPResponseCode(error), body: { message: error.message }, - }); + }; } // handle any older Boom-based errors or the few places our app uses them if (isBoom(error)) { // only log the message logger.error(error.output.payload.message); - return response.customError({ + return { statusCode: error.output.statusCode, body: { message: error.output.payload.message }, - }); + }; } // not sure what type of error this is. log as much as possible logger.error(error); - return response.customError({ + return { statusCode: 500, body: { message: error.message }, - }); + }; +} + +export const defaultIngestErrorHandler: IngestErrorHandler = async ({ + error, + response, +}: IngestErrorHandlerParams): Promise => { + return response.customError(handleIngestError(error)); }; diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index c40e0e4ac5c0b..9d41e7679ff91 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -5,7 +5,6 @@ */ import { TypeOf } from '@kbn/config-schema'; import { RequestHandler, CustomHttpResponseOptions } from 'src/core/server'; -import { appContextService } from '../../services'; import { GetInfoResponse, InstallPackageResponse, @@ -14,6 +13,7 @@ import { GetCategoriesResponse, GetPackagesResponse, GetLimitedPackagesResponse, + UpgradePackagesResponse, } from '../../../common'; import { GetCategoriesRequestSchema, @@ -23,6 +23,7 @@ import { InstallPackageFromRegistryRequestSchema, InstallPackageByUploadRequestSchema, DeletePackageRequestSchema, + BulkUpgradePackagesFromRegistryRequestSchema, } from '../../types'; import { getCategories, @@ -34,9 +35,9 @@ import { getLimitedPackages, getInstallationObject, } from '../../services/epm/packages'; -import { IngestManagerError, defaultIngestErrorHandler } from '../../errors'; +import { defaultIngestErrorHandler } from '../../errors'; import { splitPkgKey } from '../../services/epm/registry'; -import { getInstallType } from '../../services/epm/packages/install'; +import { handleInstallPackageFailure, upgradePackages } from '../../services/epm/packages/install'; export const getCategoriesHandler: RequestHandler< undefined, @@ -136,13 +137,11 @@ export const installPackageFromRegistryHandler: RequestHandler< undefined, TypeOf > = async (context, request, response) => { - const logger = appContextService.getLogger(); const savedObjectsClient = context.core.savedObjects.client; const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; const { pkgkey } = request.params; const { pkgName, pkgVersion } = splitPkgKey(pkgkey); const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName }); - const installType = getInstallType({ pkgVersion, installedPkg }); try { const res = await installPackage({ savedObjectsClient, @@ -155,36 +154,38 @@ export const installPackageFromRegistryHandler: RequestHandler< }; return response.ok({ body }); } catch (e) { - // could have also done `return defaultIngestErrorHandler({ error: e, response })` at each of the returns, - // but doing it this way will log the outer/install errors before any inner/rollback errors const defaultResult = await defaultIngestErrorHandler({ error: e, response }); - if (e instanceof IngestManagerError) { - return defaultResult; - } + await handleInstallPackageFailure( + savedObjectsClient, + e, + pkgName, + pkgVersion, + installedPkg, + callCluster + ); - // if there is an unknown server error, uninstall any package assets or reinstall the previous version if update - try { - if (installType === 'install' || installType === 'reinstall') { - logger.error(`uninstalling ${pkgkey} after error installing`); - await removeInstallation({ savedObjectsClient, pkgkey, callCluster }); - } - if (installType === 'update') { - // @ts-ignore getInstallType ensures we have installedPkg - const prevVersion = `${pkgName}-${installedPkg.attributes.version}`; - logger.error(`rolling back to ${prevVersion} after error installing ${pkgkey}`); - await installPackage({ - savedObjectsClient, - pkgkey: prevVersion, - callCluster, - }); - } - } catch (error) { - logger.error(`failed to uninstall or rollback package after installation error ${error}`); - } return defaultResult; } }; +export const upgradePackagesFromRegistryHandler: RequestHandler< + undefined, + undefined, + TypeOf +> = async (context, request, response) => { + const savedObjectsClient = context.core.savedObjects.client; + const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; + const res = await upgradePackages({ + savedObjectsClient, + callCluster, + packagesToUpgrade: request.body.upgrade, + }); + const body: UpgradePackagesResponse = { + response: res, + }; + return response.ok({ body }); +}; + export const installPackageByUploadHandler: RequestHandler< undefined, undefined, diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/index.ts b/x-pack/plugins/ingest_manager/server/routes/epm/index.ts index 9048652f0e8a9..a7f2bd8f0c505 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/index.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/index.ts @@ -14,6 +14,7 @@ import { installPackageFromRegistryHandler, installPackageByUploadHandler, deletePackageHandler, + upgradePackagesFromRegistryHandler, } from './handlers'; import { GetCategoriesRequestSchema, @@ -23,6 +24,7 @@ import { InstallPackageFromRegistryRequestSchema, InstallPackageByUploadRequestSchema, DeletePackageRequestSchema, + BulkUpgradePackagesFromRegistryRequestSchema, } from '../../types'; const MAX_FILE_SIZE_BYTES = 104857600; // 100MB @@ -82,6 +84,15 @@ export const registerRoutes = (router: IRouter) => { installPackageFromRegistryHandler ); + router.post( + { + path: EPM_API_ROUTES.BULK_UPGRADE_PATTERN, + validate: BulkUpgradePackagesFromRegistryRequestSchema, + options: { tags: [`access:${PLUGIN_ID}-all`] }, + }, + upgradePackagesFromRegistryHandler + ); + router.post( { path: EPM_API_ROUTES.INSTALL_BY_UPLOAD_PATTERN, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 4179e82d6ad1d..f619536040e42 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -6,6 +6,8 @@ import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; import semver from 'semver'; +import Boom from 'boom'; +import { UpgradePackageError, UpgradePackageInfo } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE, MAX_TIME_COMPLETE_INSTALL } from '../../../constants'; import { AssetReference, @@ -17,6 +19,7 @@ import { EsAssetReference, ElasticsearchAssetType, InstallType, + RegistrySearchResult, } from '../../../types'; import { installIndexPatterns } from '../kibana/index_pattern/install'; import * as Registry from '../registry'; @@ -32,11 +35,12 @@ import { ArchiveAsset, } from '../kibana/assets/install'; import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; -import { deleteKibanaSavedObjectsAssets } from './remove'; -import { PackageOutdatedError } from '../../../errors'; +import { deleteKibanaSavedObjectsAssets, removeInstallation } from './remove'; +import { IngestManagerError, PackageOutdatedError } from '../../../errors'; import { getPackageSavedObjects } from './get'; import { installTransformForDataset } from '../elasticsearch/transform/install'; import { appContextService } from '../../app_context'; +import { handleIngestError } from '../../../errors/handlers'; export async function installLatestPackage(options: { savedObjectsClient: SavedObjectsClientContract; @@ -95,6 +99,117 @@ export async function ensureInstalledPackage(options: { return installation; } +export async function handleInstallPackageFailure( + savedObjectsClient: SavedObjectsClientContract, + error: IngestManagerError | Boom | Error, + pkgName: string, + pkgVersion: string, + installedPkg: SavedObject | undefined, + callCluster: CallESAsCurrentUser +) { + if (error instanceof IngestManagerError) { + return; + } + const logger = appContextService.getLogger(); + const pkgkey = Registry.pkgToPkgKey({ + name: pkgName, + version: pkgVersion, + }); + + // if there is an unknown server error, uninstall any package assets or reinstall the previous version if update + try { + const installType = getInstallType({ pkgVersion, installedPkg }); + if (installType === 'install' || installType === 'reinstall') { + logger.error(`uninstalling ${pkgkey} after error installing`); + await removeInstallation({ savedObjectsClient, pkgkey, callCluster }); + } + + if (installType === 'update') { + if (!installedPkg) { + logger.error( + `failed to rollback package after installation error ${error} because saved object was undefined` + ); + return; + } + const prevVersion = `${pkgName}-${installedPkg.attributes.version}`; + logger.error(`rolling back to ${prevVersion} after error installing ${pkgkey}`); + await installPackage({ + savedObjectsClient, + pkgkey: prevVersion, + callCluster, + }); + } + } catch (e) { + logger.error(`failed to uninstall or rollback package after installation error ${e}`); + } +} + +export async function upgradePackages({ + savedObjectsClient, + packagesToUpgrade, + callCluster, +}: { + savedObjectsClient: SavedObjectsClientContract; + packagesToUpgrade: string[]; + callCluster: CallESAsCurrentUser; +}): Promise> { + const res: Array = []; + + for (const pkgToUpgrade of packagesToUpgrade) { + let installedPkg: SavedObject | undefined; + let latestPackage: RegistrySearchResult | undefined; + try { + [installedPkg, latestPackage] = await Promise.all([ + getInstallationObject({ savedObjectsClient, pkgName: pkgToUpgrade }), + Registry.fetchFindLatestPackage(pkgToUpgrade), + ]); + } catch (e) { + res.push({ name: pkgToUpgrade, ...handleIngestError(e) }); + continue; + } + + // TODO I think we want version here and not `install_version`? + if (!installedPkg || semver.gt(latestPackage.version, installedPkg.attributes.version)) { + const pkgkey = Registry.pkgToPkgKey({ + name: latestPackage.name, + version: latestPackage.version, + }); + + try { + const assets = await installPackage({ savedObjectsClient, pkgkey, callCluster }); + res.push({ + name: pkgToUpgrade, + newVersion: latestPackage.version, + oldVersion: installedPkg?.attributes.version ?? null, + assets, + }); + } catch (e) { + res.push({ name: pkgToUpgrade, ...handleIngestError(e) }); + await handleInstallPackageFailure( + savedObjectsClient, + e, + latestPackage.name, + latestPackage.version, + installedPkg, + callCluster + ); + } + } else { + // package was already at the latest version + res.push({ + name: pkgToUpgrade, + newVersion: latestPackage.version, + oldVersion: latestPackage.version, + assets: [ + ...installedPkg.attributes.installed_es, + ...installedPkg.attributes.installed_kibana, + ], + }); + } + } + return res; +} + export async function installPackage({ savedObjectsClient, pkgkey, diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/epm.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/epm.ts index d7a801feec34f..c5ebb84affac2 100644 --- a/x-pack/plugins/ingest_manager/server/types/rest_spec/epm.ts +++ b/x-pack/plugins/ingest_manager/server/types/rest_spec/epm.ts @@ -43,6 +43,12 @@ export const InstallPackageFromRegistryRequestSchema = { ), }; +export const BulkUpgradePackagesFromRegistryRequestSchema = { + body: schema.object({ + upgrade: schema.arrayOf(schema.string(), { minSize: 1 }), + }), +}; + export const InstallPackageByUploadRequestSchema = { body: schema.buffer(), }; diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts new file mode 100644 index 0000000000000..e021f52a8ef9c --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../../helpers'; +import { + UpgradePackagesResponse, + UpgradePackageInfo, + UpgradePackageError, +} from '../../../../plugins/ingest_manager/common'; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertest = getService('supertest'); + + const deletePackage = async (pkgkey: string) => { + await supertest.delete(`/api/ingest_manager/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx'); + }; + + describe('bulk package upgrade api', async () => { + skipIfNoDockerRegistry(providerContext); + + describe('bulk package upgrade with a package already installed', async () => { + beforeEach(async () => { + await supertest + .post(`/api/ingest_manager/epm/packages/multiple_versions-0.1.0`) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + }); + afterEach(async () => { + await deletePackage('multiple_versions-0.1.0'); + await deletePackage('multiple_versions-0.3.0'); + await deletePackage('overrides-0.1.0'); + }); + + it('should return 400 if no packages are requested for upgrade', async function () { + await supertest + .post(`/api/ingest_manager/epm/packages_bulk`) + .set('kbn-xsrf', 'xxxx') + .expect(400); + }); + it('should return 200 and an array for upgrading a package', async function () { + const { body }: { body: UpgradePackagesResponse } = await supertest + .post(`/api/ingest_manager/epm/packages_bulk`) + .set('kbn-xsrf', 'xxxx') + .send({ upgrade: ['multiple_versions'] }) + .expect(200); + expect(body.response.length).equal(1); + expect(body.response[0].name).equal('multiple_versions'); + const info = body.response[0] as UpgradePackageInfo; + expect(info.oldVersion).equal('0.1.0'); + expect(info.newVersion).equal('0.3.0'); + }); + it('should the same package multiple times for upgrade', async function () { + const { body }: { body: UpgradePackagesResponse } = await supertest + .post(`/api/ingest_manager/epm/packages_bulk`) + .set('kbn-xsrf', 'xxxx') + .send({ upgrade: ['multiple_versions', 'multiple_versions'] }) + .expect(200); + expect(body.response.length).equal(2); + expect(body.response[0].name).equal('multiple_versions'); + let info = body.response[0] as UpgradePackageInfo; + expect(info.oldVersion).equal('0.1.0'); + expect(info.newVersion).equal('0.3.0'); + + info = body.response[1] as UpgradePackageInfo; + expect(info.oldVersion).equal('0.3.0'); + expect(info.newVersion).equal('0.3.0'); + expect(info.name).equal('multiple_versions'); + }); + it('should return an error for packages that do not exist', async function () { + const { body }: { body: UpgradePackagesResponse } = await supertest + .post(`/api/ingest_manager/epm/packages_bulk`) + .set('kbn-xsrf', 'xxxx') + .send({ upgrade: ['multiple_versions', 'blahblah'] }) + .expect(200); + expect(body.response.length).equal(2); + expect(body.response[0].name).equal('multiple_versions'); + const info = body.response[0] as UpgradePackageInfo; + expect(info.oldVersion).equal('0.1.0'); + expect(info.newVersion).equal('0.3.0'); + + const err = body.response[1] as UpgradePackageError; + expect(err.statusCode).equal(404); + expect(info.name).equal('blahblah'); + }); + it('should upgrade multiple packages', async function () { + const { body }: { body: UpgradePackagesResponse } = await supertest + .post(`/api/ingest_manager/epm/packages_bulk`) + .set('kbn-xsrf', 'xxxx') + .send({ upgrade: ['multiple_versions', 'overrides'] }) + .expect(200); + expect(body.response.length).equal(2); + expect(body.response[0].name).equal('multiple_versions'); + let info = body.response[0] as UpgradePackageInfo; + expect(info.oldVersion).equal('0.1.0'); + expect(info.newVersion).equal('0.3.0'); + + info = body.response[1] as UpgradePackageInfo; + expect(info.oldVersion).equal(null); + expect(info.newVersion).equal('0.1.0'); + expect(info.name).equal('overrides'); + }); + }); + + describe('bulk upgrade without package already installed', async () => { + afterEach(async () => { + await deletePackage('multiple_versions-0.3.0'); + }); + + it('should return 200 and an array for upgrading a package', async function () { + const { body }: { body: UpgradePackagesResponse } = await supertest + .post(`/api/ingest_manager/epm/packages_bulk`) + .set('kbn-xsrf', 'xxxx') + .send({ upgrade: ['multiple_versions'] }) + .expect(200); + expect(body.response.length).equal(1); + expect(body.response[0].name).equal('multiple_versions'); + const info = body.response[0] as UpgradePackageInfo; + expect(info.oldVersion).equal(null); + expect(info.newVersion).equal('0.3.0'); + }); + }); + }); +} diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/index.js b/x-pack/test/ingest_manager_api_integration/apis/epm/index.js index 28743ee5f43c2..e509babc9828b 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/index.js +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/index.js @@ -16,6 +16,7 @@ export default function loadTests({ loadTestFile }) { loadTestFile(require.resolve('./install_prerelease')); loadTestFile(require.resolve('./install_remove_assets')); loadTestFile(require.resolve('./install_update')); + loadTestFile(require.resolve('./bulk_upgrade')); loadTestFile(require.resolve('./update_assets')); loadTestFile(require.resolve('./data_stream')); loadTestFile(require.resolve('./package_install_complete')); From 6dea60605f429d759b2c9433462dee36bf2d65d1 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 18 Sep 2020 14:33:11 -0400 Subject: [PATCH 02/10] Addressing comments --- .../ingest_manager/common/constants/routes.ts | 4 +- .../common/types/rest_spec/epm.ts | 12 ++- .../ingest_manager/server/errors/handlers.ts | 58 ++++++++++++-- .../server/routes/epm/handlers.ts | 21 ++--- .../ingest_manager/server/routes/epm/index.ts | 6 +- .../server/services/epm/packages/install.ts | 52 +++++++------ .../apis/epm/bulk_upgrade.ts | 78 +++++++++---------- 7 files changed, 143 insertions(+), 88 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/constants/routes.ts b/x-pack/plugins/ingest_manager/common/constants/routes.ts index 41b877f476bfc..378a6c6c12159 100644 --- a/x-pack/plugins/ingest_manager/common/constants/routes.ts +++ b/x-pack/plugins/ingest_manager/common/constants/routes.ts @@ -14,12 +14,12 @@ export const FLEET_API_ROOT = `${API_ROOT}/fleet`; export const LIMITED_CONCURRENCY_ROUTE_TAG = 'ingest:limited-concurrency'; // EPM API routes -const EPM_PACKAGES_BULK = `${EPM_API_ROOT}/packages_bulk`; const EPM_PACKAGES_MANY = `${EPM_API_ROOT}/packages`; +const EPM_PACKAGES_BULK = `${EPM_PACKAGES_MANY}/_bulk`; const EPM_PACKAGES_ONE = `${EPM_PACKAGES_MANY}/{pkgkey}`; const EPM_PACKAGES_FILE = `${EPM_PACKAGES_MANY}/{pkgName}/{pkgVersion}`; export const EPM_API_ROUTES = { - BULK_UPGRADE_PATTERN: EPM_PACKAGES_BULK, + BULK_INSTALL_PATTERN: EPM_PACKAGES_BULK, LIST_PATTERN: EPM_PACKAGES_MANY, LIMITED_LIST_PATTERN: `${EPM_PACKAGES_MANY}/limited`, INFO_PATTERN: EPM_PACKAGES_ONE, diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts index d3459dacad596..ebcced76322c2 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts @@ -71,15 +71,13 @@ export interface InstallPackageResponse { response: AssetReference[]; } -export interface UpgradePackageError { +export interface IBulkInstallPackageError { name: string; statusCode: number; - body: { - message: string; - }; + error: string | Error; } -export interface UpgradePackageInfo { +export interface BulkInstallPackageInfo { name: string; newVersion: string; // this will be null if no package was present before the upgrade (aka it was an install) @@ -87,8 +85,8 @@ export interface UpgradePackageInfo { assets: AssetReference[]; } -export interface UpgradePackagesResponse { - response: Array; +export interface BulkInstallPackagesResponse { + response: Array; } export interface MessageResponse { diff --git a/x-pack/plugins/ingest_manager/server/errors/handlers.ts b/x-pack/plugins/ingest_manager/server/errors/handlers.ts index 8998674754dde..4f130376f44db 100644 --- a/x-pack/plugins/ingest_manager/server/errors/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/errors/handlers.ts @@ -14,6 +14,7 @@ import { import { errors as LegacyESErrors } from 'elasticsearch'; import { appContextService } from '../services'; import { IngestManagerError, RegistryError, PackageNotFoundError } from './index'; +import { IBulkInstallPackageError } from '../../common'; type IngestErrorHandler = ( params: IngestErrorHandlerParams @@ -56,7 +57,9 @@ const getHTTPResponseCode = (error: IngestManagerError): number => { return 400; // Bad Request }; -export function handleIngestError(error: IngestManagerError | Boom | Error) { +export function formatBulkInstallError( + error: IngestManagerError | Boom | Error +): Pick { const logger = appContextService.getLogger(); if (isLegacyESClientError(error)) { // there was a problem communicating with ES (e.g. via `callCluster`) @@ -71,7 +74,7 @@ export function handleIngestError(error: IngestManagerError | Boom | Error) { return { statusCode: error?.statusCode || error.status, - body: { message }, + error: message, }; } @@ -81,7 +84,7 @@ export function handleIngestError(error: IngestManagerError | Boom | Error) { logger.error(error.message); return { statusCode: getHTTPResponseCode(error), - body: { message: error.message }, + error: error.message, }; } @@ -91,7 +94,7 @@ export function handleIngestError(error: IngestManagerError | Boom | Error) { logger.error(error.output.payload.message); return { statusCode: error.output.statusCode, - body: { message: error.output.payload.message }, + error: error.output.payload.message, }; } @@ -99,7 +102,7 @@ export function handleIngestError(error: IngestManagerError | Boom | Error) { logger.error(error); return { statusCode: 500, - body: { message: error.message }, + error, }; } @@ -107,5 +110,48 @@ export const defaultIngestErrorHandler: IngestErrorHandler = async ({ error, response, }: IngestErrorHandlerParams): Promise => { - return response.customError(handleIngestError(error)); + const logger = appContextService.getLogger(); + if (isLegacyESClientError(error)) { + // there was a problem communicating with ES (e.g. via `callCluster`) + // only log the message + const message = + error?.path && error?.response + ? // if possible, return the failing endpoint and its response + `${error.message} response from ${error.path}: ${error.response}` + : error.message; + + logger.error(message); + + return response.customError({ + statusCode: error?.statusCode || error.status, + body: { message }, + }); + } + + // our "expected" errors + if (error instanceof IngestManagerError) { + // only log the message + logger.error(error.message); + return response.customError({ + statusCode: getHTTPResponseCode(error), + body: { message: error.message }, + }); + } + + // handle any older Boom-based errors or the few places our app uses them + if (isBoom(error)) { + // only log the message + logger.error(error.output.payload.message); + return response.customError({ + statusCode: error.output.statusCode, + body: { message: error.output.payload.message }, + }); + } + + // not sure what type of error this is. log as much as possible + logger.error(error); + return response.customError({ + statusCode: 500, + body: { message: error.message }, + }); }; diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index 9d41e7679ff91..ed3f3077cd4e6 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -13,7 +13,7 @@ import { GetCategoriesResponse, GetPackagesResponse, GetLimitedPackagesResponse, - UpgradePackagesResponse, + BulkInstallPackagesResponse, } from '../../../common'; import { GetCategoriesRequestSchema, @@ -37,7 +37,10 @@ import { } from '../../services/epm/packages'; import { defaultIngestErrorHandler } from '../../errors'; import { splitPkgKey } from '../../services/epm/registry'; -import { handleInstallPackageFailure, upgradePackages } from '../../services/epm/packages/install'; +import { + handleInstallPackageFailure, + bulkInstallPackages, +} from '../../services/epm/packages/install'; export const getCategoriesHandler: RequestHandler< undefined, @@ -155,32 +158,32 @@ export const installPackageFromRegistryHandler: RequestHandler< return response.ok({ body }); } catch (e) { const defaultResult = await defaultIngestErrorHandler({ error: e, response }); - await handleInstallPackageFailure( + await handleInstallPackageFailure({ savedObjectsClient, - e, + error: e, pkgName, pkgVersion, installedPkg, - callCluster - ); + callCluster, + }); return defaultResult; } }; -export const upgradePackagesFromRegistryHandler: RequestHandler< +export const bulkInstallPackagesFromRegistryHandler: RequestHandler< undefined, undefined, TypeOf > = async (context, request, response) => { const savedObjectsClient = context.core.savedObjects.client; const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser; - const res = await upgradePackages({ + const res = await bulkInstallPackages({ savedObjectsClient, callCluster, packagesToUpgrade: request.body.upgrade, }); - const body: UpgradePackagesResponse = { + const body: BulkInstallPackagesResponse = { response: res, }; return response.ok({ body }); diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/index.ts b/x-pack/plugins/ingest_manager/server/routes/epm/index.ts index a7f2bd8f0c505..eaf61335b5e06 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/index.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/index.ts @@ -14,7 +14,7 @@ import { installPackageFromRegistryHandler, installPackageByUploadHandler, deletePackageHandler, - upgradePackagesFromRegistryHandler, + bulkInstallPackagesFromRegistryHandler, } from './handlers'; import { GetCategoriesRequestSchema, @@ -86,11 +86,11 @@ export const registerRoutes = (router: IRouter) => { router.post( { - path: EPM_API_ROUTES.BULK_UPGRADE_PATTERN, + path: EPM_API_ROUTES.BULK_INSTALL_PATTERN, validate: BulkUpgradePackagesFromRegistryRequestSchema, options: { tags: [`access:${PLUGIN_ID}-all`] }, }, - upgradePackagesFromRegistryHandler + bulkInstallPackagesFromRegistryHandler ); router.post( diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index f619536040e42..cdc30a3b18c15 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -7,7 +7,7 @@ import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; import semver from 'semver'; import Boom from 'boom'; -import { UpgradePackageError, UpgradePackageInfo } from '../../../../common'; +import { BulkInstallPackageInfo, IBulkInstallPackageError } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE, MAX_TIME_COMPLETE_INSTALL } from '../../../constants'; import { AssetReference, @@ -40,7 +40,7 @@ import { IngestManagerError, PackageOutdatedError } from '../../../errors'; import { getPackageSavedObjects } from './get'; import { installTransformForDataset } from '../elasticsearch/transform/install'; import { appContextService } from '../../app_context'; -import { handleIngestError } from '../../../errors/handlers'; +import { formatBulkInstallError } from '../../../errors/handlers'; export async function installLatestPackage(options: { savedObjectsClient: SavedObjectsClientContract; @@ -99,14 +99,21 @@ export async function ensureInstalledPackage(options: { return installation; } -export async function handleInstallPackageFailure( - savedObjectsClient: SavedObjectsClientContract, - error: IngestManagerError | Boom | Error, - pkgName: string, - pkgVersion: string, - installedPkg: SavedObject | undefined, - callCluster: CallESAsCurrentUser -) { +export async function handleInstallPackageFailure({ + savedObjectsClient, + error, + pkgName, + pkgVersion, + installedPkg, + callCluster, +}: { + savedObjectsClient: SavedObjectsClientContract; + error: IngestManagerError | Boom | Error; + pkgName: string; + pkgVersion: string; + installedPkg: SavedObject | undefined; + callCluster: CallESAsCurrentUser; +}) { if (error instanceof IngestManagerError) { return; } @@ -144,7 +151,9 @@ export async function handleInstallPackageFailure( } } -export async function upgradePackages({ +type BulkInstallResponse = BulkInstallPackageInfo | IBulkInstallPackageError; + +export async function bulkInstallPackages({ savedObjectsClient, packagesToUpgrade, callCluster, @@ -152,9 +161,8 @@ export async function upgradePackages({ savedObjectsClient: SavedObjectsClientContract; packagesToUpgrade: string[]; callCluster: CallESAsCurrentUser; -}): Promise> { - const res: Array = []; - +}): Promise { + const res: BulkInstallResponse[] = []; for (const pkgToUpgrade of packagesToUpgrade) { let installedPkg: SavedObject | undefined; let latestPackage: RegistrySearchResult | undefined; @@ -164,7 +172,7 @@ export async function upgradePackages({ Registry.fetchFindLatestPackage(pkgToUpgrade), ]); } catch (e) { - res.push({ name: pkgToUpgrade, ...handleIngestError(e) }); + res.push({ name: pkgToUpgrade, ...formatBulkInstallError(e) }); continue; } @@ -184,15 +192,15 @@ export async function upgradePackages({ assets, }); } catch (e) { - res.push({ name: pkgToUpgrade, ...handleIngestError(e) }); - await handleInstallPackageFailure( + res.push({ name: pkgToUpgrade, ...formatBulkInstallError(e) }); + await handleInstallPackageFailure({ savedObjectsClient, - e, - latestPackage.name, - latestPackage.version, + error: e, + pkgName: latestPackage.name, + pkgVersion: latestPackage.version, installedPkg, - callCluster - ); + callCluster, + }); } } else { // package was already at the latest version diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts index e021f52a8ef9c..acd538cc71173 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts @@ -8,9 +8,9 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; import { - UpgradePackagesResponse, - UpgradePackageInfo, - UpgradePackageError, + BulkInstallPackageInfo, + BulkInstallPackagesResponse, + IBulkInstallPackageError, } from '../../../../plugins/ingest_manager/common'; export default function (providerContext: FtrProviderContext) { @@ -40,71 +40,71 @@ export default function (providerContext: FtrProviderContext) { it('should return 400 if no packages are requested for upgrade', async function () { await supertest - .post(`/api/ingest_manager/epm/packages_bulk`) + .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') .expect(400); }); it('should return 200 and an array for upgrading a package', async function () { - const { body }: { body: UpgradePackagesResponse } = await supertest - .post(`/api/ingest_manager/epm/packages_bulk`) + const { body }: { body: BulkInstallPackagesResponse } = await supertest + .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') .send({ upgrade: ['multiple_versions'] }) .expect(200); expect(body.response.length).equal(1); expect(body.response[0].name).equal('multiple_versions'); - const info = body.response[0] as UpgradePackageInfo; - expect(info.oldVersion).equal('0.1.0'); - expect(info.newVersion).equal('0.3.0'); + const entry = body.response[0] as BulkInstallPackageInfo; + expect(entry.oldVersion).equal('0.1.0'); + expect(entry.newVersion).equal('0.3.0'); }); it('should the same package multiple times for upgrade', async function () { - const { body }: { body: UpgradePackagesResponse } = await supertest - .post(`/api/ingest_manager/epm/packages_bulk`) + const { body }: { body: BulkInstallPackagesResponse } = await supertest + .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') .send({ upgrade: ['multiple_versions', 'multiple_versions'] }) .expect(200); expect(body.response.length).equal(2); expect(body.response[0].name).equal('multiple_versions'); - let info = body.response[0] as UpgradePackageInfo; - expect(info.oldVersion).equal('0.1.0'); - expect(info.newVersion).equal('0.3.0'); + let entry = body.response[0] as BulkInstallPackageInfo; + expect(entry.oldVersion).equal('0.1.0'); + expect(entry.newVersion).equal('0.3.0'); - info = body.response[1] as UpgradePackageInfo; - expect(info.oldVersion).equal('0.3.0'); - expect(info.newVersion).equal('0.3.0'); - expect(info.name).equal('multiple_versions'); + entry = body.response[1] as BulkInstallPackageInfo; + expect(entry.oldVersion).equal('0.3.0'); + expect(entry.newVersion).equal('0.3.0'); + expect(body.response[1].name).equal('multiple_versions'); }); it('should return an error for packages that do not exist', async function () { - const { body }: { body: UpgradePackagesResponse } = await supertest - .post(`/api/ingest_manager/epm/packages_bulk`) + const { body }: { body: BulkInstallPackagesResponse } = await supertest + .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') .send({ upgrade: ['multiple_versions', 'blahblah'] }) .expect(200); expect(body.response.length).equal(2); expect(body.response[0].name).equal('multiple_versions'); - const info = body.response[0] as UpgradePackageInfo; - expect(info.oldVersion).equal('0.1.0'); - expect(info.newVersion).equal('0.3.0'); + const entry = body.response[0] as BulkInstallPackageInfo; + expect(entry.oldVersion).equal('0.1.0'); + expect(entry.newVersion).equal('0.3.0'); - const err = body.response[1] as UpgradePackageError; + const err = body.response[1] as IBulkInstallPackageError; expect(err.statusCode).equal(404); - expect(info.name).equal('blahblah'); + expect(body.response[1].name).equal('blahblah'); }); it('should upgrade multiple packages', async function () { - const { body }: { body: UpgradePackagesResponse } = await supertest - .post(`/api/ingest_manager/epm/packages_bulk`) + const { body }: { body: BulkInstallPackagesResponse } = await supertest + .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') .send({ upgrade: ['multiple_versions', 'overrides'] }) .expect(200); expect(body.response.length).equal(2); expect(body.response[0].name).equal('multiple_versions'); - let info = body.response[0] as UpgradePackageInfo; - expect(info.oldVersion).equal('0.1.0'); - expect(info.newVersion).equal('0.3.0'); + let entry = body.response[0] as BulkInstallPackageInfo; + expect(entry.oldVersion).equal('0.1.0'); + expect(entry.newVersion).equal('0.3.0'); - info = body.response[1] as UpgradePackageInfo; - expect(info.oldVersion).equal(null); - expect(info.newVersion).equal('0.1.0'); - expect(info.name).equal('overrides'); + entry = body.response[1] as BulkInstallPackageInfo; + expect(entry.oldVersion).equal(null); + expect(entry.newVersion).equal('0.1.0'); + expect(entry.name).equal('overrides'); }); }); @@ -114,16 +114,16 @@ export default function (providerContext: FtrProviderContext) { }); it('should return 200 and an array for upgrading a package', async function () { - const { body }: { body: UpgradePackagesResponse } = await supertest - .post(`/api/ingest_manager/epm/packages_bulk`) + const { body }: { body: BulkInstallPackagesResponse } = await supertest + .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') .send({ upgrade: ['multiple_versions'] }) .expect(200); expect(body.response.length).equal(1); expect(body.response[0].name).equal('multiple_versions'); - const info = body.response[0] as UpgradePackageInfo; - expect(info.oldVersion).equal(null); - expect(info.newVersion).equal('0.3.0'); + const entry = body.response[0] as BulkInstallPackageInfo; + expect(entry.oldVersion).equal(null); + expect(entry.newVersion).equal('0.3.0'); }); }); }); From 268c5266c2b0970feff9dfb7e5335a5dbfdcd91c Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 18 Sep 2020 14:58:24 -0400 Subject: [PATCH 03/10] Removing todo --- .../ingest_manager/server/services/epm/packages/install.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index cdc30a3b18c15..0ffff8c95578b 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -176,7 +176,6 @@ export async function bulkInstallPackages({ continue; } - // TODO I think we want version here and not `install_version`? if (!installedPkg || semver.gt(latestPackage.version, installedPkg.attributes.version)) { const pkgkey = Registry.pkgToPkgKey({ name: latestPackage.name, From f11140603918e6995277a7f6aaac67ede427179b Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 18 Sep 2020 15:09:47 -0400 Subject: [PATCH 04/10] Changing body field --- .../ingest_manager/server/routes/epm/handlers.ts | 2 +- .../ingest_manager/server/types/rest_spec/epm.ts | 2 +- .../apis/epm/bulk_upgrade.ts | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index ed3f3077cd4e6..7ae896c1f30a6 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -181,7 +181,7 @@ export const bulkInstallPackagesFromRegistryHandler: RequestHandler< const res = await bulkInstallPackages({ savedObjectsClient, callCluster, - packagesToUpgrade: request.body.upgrade, + packagesToUpgrade: request.body.packages, }); const body: BulkInstallPackagesResponse = { response: res, diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/epm.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/epm.ts index c5ebb84affac2..5d2a078374854 100644 --- a/x-pack/plugins/ingest_manager/server/types/rest_spec/epm.ts +++ b/x-pack/plugins/ingest_manager/server/types/rest_spec/epm.ts @@ -45,7 +45,7 @@ export const InstallPackageFromRegistryRequestSchema = { export const BulkUpgradePackagesFromRegistryRequestSchema = { body: schema.object({ - upgrade: schema.arrayOf(schema.string(), { minSize: 1 }), + packages: schema.arrayOf(schema.string(), { minSize: 1 }), }), }; diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts index acd538cc71173..5c46ea813d997 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts @@ -48,7 +48,7 @@ export default function (providerContext: FtrProviderContext) { const { body }: { body: BulkInstallPackagesResponse } = await supertest .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') - .send({ upgrade: ['multiple_versions'] }) + .send({ packages: ['multiple_versions'] }) .expect(200); expect(body.response.length).equal(1); expect(body.response[0].name).equal('multiple_versions'); @@ -60,7 +60,7 @@ export default function (providerContext: FtrProviderContext) { const { body }: { body: BulkInstallPackagesResponse } = await supertest .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') - .send({ upgrade: ['multiple_versions', 'multiple_versions'] }) + .send({ packages: ['multiple_versions', 'multiple_versions'] }) .expect(200); expect(body.response.length).equal(2); expect(body.response[0].name).equal('multiple_versions'); @@ -77,7 +77,7 @@ export default function (providerContext: FtrProviderContext) { const { body }: { body: BulkInstallPackagesResponse } = await supertest .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') - .send({ upgrade: ['multiple_versions', 'blahblah'] }) + .send({ packages: ['multiple_versions', 'blahblah'] }) .expect(200); expect(body.response.length).equal(2); expect(body.response[0].name).equal('multiple_versions'); @@ -93,7 +93,7 @@ export default function (providerContext: FtrProviderContext) { const { body }: { body: BulkInstallPackagesResponse } = await supertest .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') - .send({ upgrade: ['multiple_versions', 'overrides'] }) + .send({ packages: ['multiple_versions', 'overrides'] }) .expect(200); expect(body.response.length).equal(2); expect(body.response[0].name).equal('multiple_versions'); @@ -117,7 +117,7 @@ export default function (providerContext: FtrProviderContext) { const { body }: { body: BulkInstallPackagesResponse } = await supertest .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') - .send({ upgrade: ['multiple_versions'] }) + .send({ packages: ['multiple_versions'] }) .expect(200); expect(body.response.length).equal(1); expect(body.response[0].name).equal('multiple_versions'); From 2846b761debbacf5b958daf6cc44d69eb36a1881 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 18 Sep 2020 15:24:22 -0400 Subject: [PATCH 05/10] Adding helper for getting the bulk install route --- x-pack/plugins/ingest_manager/common/services/routes.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/ingest_manager/common/services/routes.ts b/x-pack/plugins/ingest_manager/common/services/routes.ts index b7521f95b4f83..ec7c0ee850834 100644 --- a/x-pack/plugins/ingest_manager/common/services/routes.ts +++ b/x-pack/plugins/ingest_manager/common/services/routes.ts @@ -46,6 +46,10 @@ export const epmRouteService = { ); // trim trailing slash }, + getBulkInstallPath: () => { + return EPM_API_ROUTES.BULK_INSTALL_PATTERN; + }, + getRemovePath: (pkgkey: string) => { return EPM_API_ROUTES.DELETE_PATTERN.replace('{pkgkey}', pkgkey).replace(/\/$/, ''); // trim trailing slash }, From 20357fd2c800ede81394e96cc3bee0a26d825f71 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Fri, 18 Sep 2020 15:35:11 -0400 Subject: [PATCH 06/10] Adding request spec --- x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts index ebcced76322c2..7ed2fed91aa93 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/epm.ts @@ -89,6 +89,12 @@ export interface BulkInstallPackagesResponse { response: Array; } +export interface BulkInstallPackagesRequest { + body: { + packages: string[]; + }; +} + export interface MessageResponse { response: string; } From 9b29802f97a37b12795fbcb5b05d40301ae8080e Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 21 Sep 2020 10:32:53 -0400 Subject: [PATCH 07/10] Pulling in Johns changes --- .../server/services/epm/packages/install.ts | 172 +++++++++++------- .../apis/epm/bulk_upgrade.ts | 19 +- 2 files changed, 112 insertions(+), 79 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 15a602182a5ba..5c20d4185bab3 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -7,6 +7,7 @@ import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; import semver from 'semver'; import Boom from 'boom'; +import { UnwrapPromise } from '@kbn/utility-types'; import { BulkInstallPackageInfo, IBulkInstallPackageError } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE, MAX_TIME_COMPLETE_INSTALL } from '../../../constants'; import { @@ -19,7 +20,6 @@ import { EsAssetReference, ElasticsearchAssetType, InstallType, - RegistrySearchResult, } from '../../../types'; import { installIndexPatterns } from '../kibana/index_pattern/install'; import * as Registry from '../registry'; @@ -152,82 +152,132 @@ export async function handleInstallPackageFailure({ } type BulkInstallResponse = BulkInstallPackageInfo | IBulkInstallPackageError; - -export async function bulkInstallPackages({ - savedObjectsClient, - packagesToUpgrade, - callCluster, +function bulkInstallErrorToOptions({ + pkgToUpgrade, + error, }: { + pkgToUpgrade: string; + error: Error; +}): IBulkInstallPackageError { + const { statusCode, error: err } = formatBulkInstallError(error); + return { + name: pkgToUpgrade, + statusCode, + error: err, + }; +} + +interface UpgradePackageParams { savedObjectsClient: SavedObjectsClientContract; - packagesToUpgrade: string[]; callCluster: CallESAsCurrentUser; -}): Promise { - const res: BulkInstallResponse[] = []; - for (const pkgToUpgrade of packagesToUpgrade) { - let installedPkg: SavedObject | undefined; - let latestPackage: RegistrySearchResult | undefined; - try { - [installedPkg, latestPackage] = await Promise.all([ - getInstallationObject({ savedObjectsClient, pkgName: pkgToUpgrade }), - Registry.fetchFindLatestPackage(pkgToUpgrade), - ]); - } catch (e) { - res.push({ name: pkgToUpgrade, ...formatBulkInstallError(e) }); - continue; - } - - if (!installedPkg || semver.gt(latestPackage.version, installedPkg.attributes.version)) { - const pkgkey = Registry.pkgToPkgKey({ - name: latestPackage.name, - version: latestPackage.version, - }); + installedPkg: UnwrapPromise>; + latestPkg: UnwrapPromise>; + pkgToUpgrade: string; +} +async function upgradePackage({ + savedObjectsClient, + callCluster, + installedPkg, + latestPkg, + pkgToUpgrade, +}: UpgradePackageParams): Promise { + if (!installedPkg || semver.gt(latestPkg.version, installedPkg.attributes.version)) { + const pkgkey = Registry.pkgToPkgKey({ + name: latestPkg.name, + version: latestPkg.version, + }); - try { - const assets = await installPackage({ savedObjectsClient, pkgkey, callCluster }); - res.push({ - name: pkgToUpgrade, - newVersion: latestPackage.version, - oldVersion: installedPkg?.attributes.version ?? null, - assets, - }); - } catch (e) { - res.push({ name: pkgToUpgrade, ...formatBulkInstallError(e) }); - await handleInstallPackageFailure({ - savedObjectsClient, - error: e, - pkgName: latestPackage.name, - pkgVersion: latestPackage.version, - installedPkg, - callCluster, - }); - } - } else { - // package was already at the latest version - res.push({ + try { + const assets = await installPackage({ savedObjectsClient, pkgkey, callCluster }); + return { name: pkgToUpgrade, - newVersion: latestPackage.version, - oldVersion: latestPackage.version, - assets: [ - ...installedPkg.attributes.installed_es, - ...installedPkg.attributes.installed_kibana, - ], + newVersion: latestPkg.version, + oldVersion: installedPkg?.attributes.version ?? null, + assets, + }; + } catch (installFailed) { + await handleInstallPackageFailure({ + savedObjectsClient, + error: installFailed, + pkgName: latestPkg.name, + pkgVersion: latestPkg.version, + installedPkg, + callCluster, }); + return bulkInstallErrorToOptions({ pkgToUpgrade, error: installFailed }); } + } else { + // package was already at the latest version + return { + name: pkgToUpgrade, + newVersion: latestPkg.version, + oldVersion: latestPkg.version, + assets: [ + ...installedPkg.attributes.installed_es, + ...installedPkg.attributes.installed_kibana, + ], + }; } - return res; } -export async function installPackage({ +interface BulkInstallPackagesParams { + savedObjectsClient: SavedObjectsClientContract; + packagesToUpgrade: string[]; + callCluster: CallESAsCurrentUser; +} +export async function bulkInstallPackages({ savedObjectsClient, - pkgkey, + packagesToUpgrade, callCluster, - force = false, -}: { +}: BulkInstallPackagesParams): Promise { + const installedAndLatestPromises = packagesToUpgrade.map((pkgToUpgrade) => + Promise.all([ + getInstallationObject({ savedObjectsClient, pkgName: pkgToUpgrade }), + Registry.fetchFindLatestPackage(pkgToUpgrade), + ]) + ); + const installedAndLatestResults = await Promise.allSettled(installedAndLatestPromises); + const installResponsePromises = installedAndLatestResults.map(async (result, index) => { + const pkgToUpgrade = packagesToUpgrade[index]; + if (result.status === 'fulfilled') { + const [installedPkg, latestPkg] = result.value; + return upgradePackage({ + savedObjectsClient, + callCluster, + installedPkg, + latestPkg, + pkgToUpgrade, + }); + } else { + return bulkInstallErrorToOptions({ pkgToUpgrade, error: result.reason }); + } + }); + const installResults = await Promise.allSettled(installResponsePromises); + const installResponses = installResults.map((result, index) => { + const pkgToUpgrade = packagesToUpgrade[index]; + if (result.status === 'fulfilled') { + return result.value; + } else { + return bulkInstallErrorToOptions({ pkgToUpgrade, error: result.reason }); + } + }); + + return installResponses; +} + +interface InstallPackageParams { savedObjectsClient: SavedObjectsClientContract; pkgkey: string; callCluster: CallESAsCurrentUser; force?: boolean; -}): Promise { +} + +export async function installPackage({ + savedObjectsClient, + pkgkey, + callCluster, + force = false, +}: InstallPackageParams): Promise { // TODO: change epm API to /packageName/version so we don't need to do this const { pkgName, pkgVersion } = Registry.splitPkgKey(pkgkey); // TODO: calls to getInstallationObject, Registry.fetchInfo, and Registry.fetchFindLatestPackge diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts index 5c46ea813d997..d2e669e587a4e 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts @@ -56,7 +56,7 @@ export default function (providerContext: FtrProviderContext) { expect(entry.oldVersion).equal('0.1.0'); expect(entry.newVersion).equal('0.3.0'); }); - it('should the same package multiple times for upgrade', async function () { + it('should handle the same package multiple times for upgrade', async function () { const { body }: { body: BulkInstallPackagesResponse } = await supertest .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') @@ -89,23 +89,6 @@ export default function (providerContext: FtrProviderContext) { expect(err.statusCode).equal(404); expect(body.response[1].name).equal('blahblah'); }); - it('should upgrade multiple packages', async function () { - const { body }: { body: BulkInstallPackagesResponse } = await supertest - .post(`/api/ingest_manager/epm/packages/_bulk`) - .set('kbn-xsrf', 'xxxx') - .send({ packages: ['multiple_versions', 'overrides'] }) - .expect(200); - expect(body.response.length).equal(2); - expect(body.response[0].name).equal('multiple_versions'); - let entry = body.response[0] as BulkInstallPackageInfo; - expect(entry.oldVersion).equal('0.1.0'); - expect(entry.newVersion).equal('0.3.0'); - - entry = body.response[1] as BulkInstallPackageInfo; - expect(entry.oldVersion).equal(null); - expect(entry.newVersion).equal('0.1.0'); - expect(entry.name).equal('overrides'); - }); }); describe('bulk upgrade without package already installed', async () => { From 7cb80e9c251d9357659770c964fe53e1c3717c18 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Mon, 21 Sep 2020 11:00:54 -0400 Subject: [PATCH 08/10] Removing test for same package upgraded multiple times --- .../apis/epm/bulk_upgrade.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts index d2e669e587a4e..e377ea5a762f9 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/bulk_upgrade.ts @@ -56,38 +56,38 @@ export default function (providerContext: FtrProviderContext) { expect(entry.oldVersion).equal('0.1.0'); expect(entry.newVersion).equal('0.3.0'); }); - it('should handle the same package multiple times for upgrade', async function () { + it('should return an error for packages that do not exist', async function () { const { body }: { body: BulkInstallPackagesResponse } = await supertest .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') - .send({ packages: ['multiple_versions', 'multiple_versions'] }) + .send({ packages: ['multiple_versions', 'blahblah'] }) .expect(200); expect(body.response.length).equal(2); expect(body.response[0].name).equal('multiple_versions'); - let entry = body.response[0] as BulkInstallPackageInfo; + const entry = body.response[0] as BulkInstallPackageInfo; expect(entry.oldVersion).equal('0.1.0'); expect(entry.newVersion).equal('0.3.0'); - entry = body.response[1] as BulkInstallPackageInfo; - expect(entry.oldVersion).equal('0.3.0'); - expect(entry.newVersion).equal('0.3.0'); - expect(body.response[1].name).equal('multiple_versions'); + const err = body.response[1] as IBulkInstallPackageError; + expect(err.statusCode).equal(404); + expect(body.response[1].name).equal('blahblah'); }); - it('should return an error for packages that do not exist', async function () { + it('should upgrade multiple packages', async function () { const { body }: { body: BulkInstallPackagesResponse } = await supertest .post(`/api/ingest_manager/epm/packages/_bulk`) .set('kbn-xsrf', 'xxxx') - .send({ packages: ['multiple_versions', 'blahblah'] }) + .send({ packages: ['multiple_versions', 'overrides'] }) .expect(200); expect(body.response.length).equal(2); expect(body.response[0].name).equal('multiple_versions'); - const entry = body.response[0] as BulkInstallPackageInfo; + let entry = body.response[0] as BulkInstallPackageInfo; expect(entry.oldVersion).equal('0.1.0'); expect(entry.newVersion).equal('0.3.0'); - const err = body.response[1] as IBulkInstallPackageError; - expect(err.statusCode).equal(404); - expect(body.response[1].name).equal('blahblah'); + entry = body.response[1] as BulkInstallPackageInfo; + expect(entry.oldVersion).equal(null); + expect(entry.newVersion).equal('0.1.0'); + expect(entry.name).equal('overrides'); }); }); From 6ef14cc69772f7a0b07d0f0881003ab204278e00 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 22 Sep 2020 10:20:28 -0400 Subject: [PATCH 09/10] Pulling in John's error handling changes --- .../ingest_manager/server/errors/handlers.ts | 58 +++---------------- .../ingest_manager/server/errors/index.ts | 2 +- .../server/services/epm/packages/install.ts | 11 ++-- 3 files changed, 15 insertions(+), 56 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/errors/handlers.ts b/x-pack/plugins/ingest_manager/server/errors/handlers.ts index 4f130376f44db..b550f11148a77 100644 --- a/x-pack/plugins/ingest_manager/server/errors/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/errors/handlers.ts @@ -57,9 +57,7 @@ const getHTTPResponseCode = (error: IngestManagerError): number => { return 400; // Bad Request }; -export function formatBulkInstallError( - error: IngestManagerError | Boom | Error -): Pick { +export function ingestErrorToResponseOptions(error: IngestErrorHandlerParams['error']) { const logger = appContextService.getLogger(); if (isLegacyESClientError(error)) { // there was a problem communicating with ES (e.g. via `callCluster`) @@ -74,7 +72,7 @@ export function formatBulkInstallError( return { statusCode: error?.statusCode || error.status, - error: message, + body: { message }, }; } @@ -84,7 +82,7 @@ export function formatBulkInstallError( logger.error(error.message); return { statusCode: getHTTPResponseCode(error), - error: error.message, + body: { message: error.message }, }; } @@ -94,7 +92,7 @@ export function formatBulkInstallError( logger.error(error.output.payload.message); return { statusCode: error.output.statusCode, - error: error.output.payload.message, + body: { message: error.output.payload.message }, }; } @@ -102,7 +100,7 @@ export function formatBulkInstallError( logger.error(error); return { statusCode: 500, - error, + body: { message: error.message }, }; } @@ -110,48 +108,6 @@ export const defaultIngestErrorHandler: IngestErrorHandler = async ({ error, response, }: IngestErrorHandlerParams): Promise => { - const logger = appContextService.getLogger(); - if (isLegacyESClientError(error)) { - // there was a problem communicating with ES (e.g. via `callCluster`) - // only log the message - const message = - error?.path && error?.response - ? // if possible, return the failing endpoint and its response - `${error.message} response from ${error.path}: ${error.response}` - : error.message; - - logger.error(message); - - return response.customError({ - statusCode: error?.statusCode || error.status, - body: { message }, - }); - } - - // our "expected" errors - if (error instanceof IngestManagerError) { - // only log the message - logger.error(error.message); - return response.customError({ - statusCode: getHTTPResponseCode(error), - body: { message: error.message }, - }); - } - - // handle any older Boom-based errors or the few places our app uses them - if (isBoom(error)) { - // only log the message - logger.error(error.output.payload.message); - return response.customError({ - statusCode: error.output.statusCode, - body: { message: error.output.payload.message }, - }); - } - - // not sure what type of error this is. log as much as possible - logger.error(error); - return response.customError({ - statusCode: 500, - body: { message: error.message }, - }); + const options = ingestErrorToResponseOptions(error); + return response.customError(options); }; diff --git a/x-pack/plugins/ingest_manager/server/errors/index.ts b/x-pack/plugins/ingest_manager/server/errors/index.ts index 5e36a2ec9a884..f495bf551dcff 100644 --- a/x-pack/plugins/ingest_manager/server/errors/index.ts +++ b/x-pack/plugins/ingest_manager/server/errors/index.ts @@ -5,7 +5,7 @@ */ /* eslint-disable max-classes-per-file */ -export { defaultIngestErrorHandler } from './handlers'; +export { defaultIngestErrorHandler, ingestErrorToResponseOptions } from './handlers'; export class IngestManagerError extends Error { constructor(message?: string) { diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 5c20d4185bab3..800151a41a429 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -36,11 +36,14 @@ import { } from '../kibana/assets/install'; import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; import { deleteKibanaSavedObjectsAssets, removeInstallation } from './remove'; -import { IngestManagerError, PackageOutdatedError } from '../../../errors'; +import { + IngestManagerError, + PackageOutdatedError, + ingestErrorToResponseOptions, +} from '../../../errors'; import { getPackageSavedObjects } from './get'; import { installTransformForDataset } from '../elasticsearch/transform/install'; import { appContextService } from '../../app_context'; -import { formatBulkInstallError } from '../../../errors/handlers'; export async function installLatestPackage(options: { savedObjectsClient: SavedObjectsClientContract; @@ -159,11 +162,11 @@ function bulkInstallErrorToOptions({ pkgToUpgrade: string; error: Error; }): IBulkInstallPackageError { - const { statusCode, error: err } = formatBulkInstallError(error); + const { statusCode, body } = ingestErrorToResponseOptions(error); return { name: pkgToUpgrade, statusCode, - error: err, + error: body.message, }; } From df323083489bee88d14288969a7c81145e9026d7 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner Date: Tue, 22 Sep 2020 10:55:28 -0400 Subject: [PATCH 10/10] Fixing type error --- x-pack/plugins/ingest_manager/server/errors/handlers.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/errors/handlers.ts b/x-pack/plugins/ingest_manager/server/errors/handlers.ts index b550f11148a77..b621f2dd29331 100644 --- a/x-pack/plugins/ingest_manager/server/errors/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/errors/handlers.ts @@ -14,7 +14,6 @@ import { import { errors as LegacyESErrors } from 'elasticsearch'; import { appContextService } from '../services'; import { IngestManagerError, RegistryError, PackageNotFoundError } from './index'; -import { IBulkInstallPackageError } from '../../common'; type IngestErrorHandler = ( params: IngestErrorHandlerParams