diff --git a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts index cb9f5650550e8..2c313f2f2761d 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/storage.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/storage.ts @@ -123,7 +123,7 @@ export async function saveArchiveEntries(opts: { }) ); - const results = await savedObjectsClient.bulkCreate(bulkBody); + const results = await savedObjectsClient.bulkCreate(bulkBody, { refresh: false }); return results; } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts index 4f18966a61307..c6be2dfedb1df 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts @@ -13,14 +13,13 @@ import type { InstallablePackage, RegistryDataStream, } from '../../../../../common/types/models'; -import { getInstallation } from '../../packages'; -import { saveInstalledEsRefs } from '../../packages/install'; +import { updateEsAssetReferences } from '../../packages/install'; import { getAsset } from '../transform/common'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; -import { deleteIlmRefs, deleteIlms } from './remove'; +import { deleteIlms } from './remove'; interface IlmInstallation { installationName: string; @@ -37,24 +36,39 @@ export const installIlmForDataStream = async ( paths: string[], esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract, - logger: Logger + logger: Logger, + esReferences: EsAssetReference[] ) => { - const installation = await getInstallation({ savedObjectsClient, pkgName: registryPackage.name }); - let previousInstalledIlmEsAssets: EsAssetReference[] = []; - if (installation) { - previousInstalledIlmEsAssets = installation.installed_es.filter( - ({ type, id }) => type === ElasticsearchAssetType.dataStreamIlmPolicy - ); - } + const previousInstalledIlmEsAssets = esReferences.filter( + ({ type }) => type === ElasticsearchAssetType.dataStreamIlmPolicy + ); // delete all previous ilm await deleteIlms( esClient, previousInstalledIlmEsAssets.map((asset) => asset.id) ); + + if (previousInstalledIlmEsAssets.length > 0) { + // remove the saved object reference + esReferences = await updateEsAssetReferences( + savedObjectsClient, + registryPackage.name, + esReferences, + { + assetsToRemove: previousInstalledIlmEsAssets, + } + ); + } + // install the latest dataset const dataStreams = registryPackage.data_streams; - if (!dataStreams?.length) return []; + if (!dataStreams?.length) + return { + installedIlms: [], + esReferences, + }; + const dataStreamIlmPaths = paths.filter((path) => isDataStreamIlm(path)); let installedIlms: EsAssetReference[] = []; if (dataStreamIlmPaths.length > 0) { @@ -77,12 +91,17 @@ export const installIlmForDataStream = async ( return acc; }, []); - await saveInstalledEsRefs(savedObjectsClient, registryPackage.name, ilmRefs); + esReferences = await updateEsAssetReferences( + savedObjectsClient, + registryPackage.name, + esReferences, + { assetsToAdd: ilmRefs } + ); const ilmInstallations: IlmInstallation[] = ilmPathDatasets.map( (ilmPathDataset: IlmPathDataset) => { const content = JSON.parse(getAsset(ilmPathDataset.path).toString('utf-8')); - content.policy._meta = getESAssetMetadata({ packageName: installation?.name }); + content.policy._meta = getESAssetMetadata({ packageName: registryPackage.name }); return { installationName: getIlmNameForInstallation(ilmPathDataset), @@ -98,22 +117,7 @@ export const installIlmForDataStream = async ( installedIlms = await Promise.all(installationPromises).then((results) => results.flat()); } - if (previousInstalledIlmEsAssets.length > 0) { - const currentInstallation = await getInstallation({ - savedObjectsClient, - pkgName: registryPackage.name, - }); - - // remove the saved object reference - await deleteIlmRefs( - savedObjectsClient, - currentInstallation?.installed_es || [], - registryPackage.name, - previousInstalledIlmEsAssets.map((asset) => asset.id), - installedIlms.map((installed) => installed.id) - ); - } - return installedIlms; + return { installedIlms, esReferences }; }; async function handleIlmInstall({ diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts index 1d98a9339c907..331088d195d0b 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/remove.ts @@ -5,11 +5,7 @@ * 2.0. */ -import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; - -import { ElasticsearchAssetType } from '../../../../types'; -import type { EsAssetReference } from '../../../../types'; -import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common/constants'; +import type { ElasticsearchClient } from '@kbn/core/server'; export const deleteIlms = async (esClient: ElasticsearchClient, ilmPolicyIds: string[]) => { await Promise.all( @@ -26,24 +22,3 @@ export const deleteIlms = async (esClient: ElasticsearchClient, ilmPolicyIds: st }) ); }; - -export const deleteIlmRefs = async ( - savedObjectsClient: SavedObjectsClientContract, - installedEsAssets: EsAssetReference[], - pkgName: string, - installedEsIdToRemove: string[], - currentInstalledEsIlmIds: string[] -) => { - const seen = new Set(); - const filteredAssets = installedEsAssets.filter(({ type, id }) => { - if (type !== ElasticsearchAssetType.dataStreamIlmPolicy) return true; - const add = - (currentInstalledEsIlmIds.includes(id) || !installedEsIdToRemove.includes(id)) && - !seen.has(id); - seen.add(id); - return add; - }); - return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { - installed_es: filteredAssets, - }); -}; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts index 9b64ec89507dc..3aa86b526addd 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ilm/install.ts @@ -5,12 +5,13 @@ * 2.0. */ -import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server'; -import type { InstallablePackage } from '../../../../types'; +import type { EsAssetReference, InstallablePackage } from '../../../../types'; import { ElasticsearchAssetType } from '../../../../types'; import { getAsset, getPathParts } from '../../archive'; +import { updateEsAssetReferences } from '../../packages/install'; import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; @@ -18,25 +19,40 @@ export async function installILMPolicy( packageInfo: InstallablePackage, paths: string[], esClient: ElasticsearchClient, - logger: Logger -) { + savedObjectsClient: SavedObjectsClientContract, + logger: Logger, + esReferences: EsAssetReference[] +): Promise { const ilmPaths = paths.filter((path) => isILMPolicy(path)); - if (!ilmPaths.length) return; - await Promise.all( - ilmPaths.map(async (path) => { - const body = JSON.parse(getAsset(path).toString('utf-8')); + if (!ilmPaths.length) return esReferences; + + const ilmPolicies = ilmPaths.map((path) => { + const body = JSON.parse(getAsset(path).toString('utf-8')); + + body.policy._meta = getESAssetMetadata({ packageName: packageInfo.name }); + + const { file } = getPathParts(path); + const name = file.substr(0, file.lastIndexOf('.')); - body.policy._meta = getESAssetMetadata({ packageName: packageInfo.name }); + return { name, body }; + }); - const { file } = getPathParts(path); - const name = file.substr(0, file.lastIndexOf('.')); + esReferences = await updateEsAssetReferences(savedObjectsClient, packageInfo.name, esReferences, { + assetsToAdd: ilmPolicies.map((policy) => ({ + type: ElasticsearchAssetType.ilmPolicy, + id: policy.name, + })), + }); + + await Promise.all( + ilmPolicies.map(async (policy) => { try { await retryTransientEsErrors( () => esClient.transport.request({ method: 'PUT', - path: '/_ilm/policy/' + name, - body, + path: '/_ilm/policy/' + policy.name, + body: policy.body, }), { logger } ); @@ -45,6 +61,8 @@ export async function installILMPolicy( } }) ); + + return esReferences; } const isILMPolicy = (path: string) => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index c6830d5bb9a03..49dae4d86b639 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -12,8 +12,7 @@ import { ElasticsearchAssetType } from '../../../../types'; import type { EsAssetReference, RegistryDataStream, InstallablePackage } from '../../../../types'; import { getAsset, getPathParts } from '../../archive'; import type { ArchiveEntry } from '../../archive'; -import { saveInstalledEsRefs } from '../../packages/install'; -import { getInstallationObject } from '../../packages'; +import { updateEsAssetReferences } from '../../packages/install'; import { FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_ID, @@ -24,8 +23,6 @@ import { appendMetadataToIngestPipeline } from '../meta'; import { retryTransientEsErrors } from '../retry'; -import { deletePipelineRefs } from './remove'; - interface RewriteSubstitution { source: string; target: string; @@ -44,7 +41,8 @@ export const installPipelines = async ( paths: string[], esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract, - logger: Logger + logger: Logger, + esReferences: EsAssetReference[] ) => { // unlike other ES assets, pipeline names are versioned so after a template is updated // it can be created pointing to the new template, without removing the old one and effecting data @@ -67,7 +65,7 @@ export const installPipelines = async ( const nameForInstallation = getPipelineNameForInstallation({ pipelineName: name, dataStream, - packageVersion: installablePackage.version, + packageVersion: pkgVersion, }); return { id: nameForInstallation, type: ElasticsearchAssetType.ingestPipeline }; }); @@ -80,27 +78,17 @@ export const installPipelines = async ( const { name } = getNameAndExtension(path); const nameForInstallation = getPipelineNameForInstallation({ pipelineName: name, - packageVersion: installablePackage.version, + packageVersion: pkgVersion, }); return { id: nameForInstallation, type: ElasticsearchAssetType.ingestPipeline }; }); pipelineRefs = [...pipelineRefs, ...topLevelPipelineRefs]; - // check that we don't duplicate the pipeline refs if the user is reinstalling - const installedPkg = await getInstallationObject({ - savedObjectsClient, - pkgName, + esReferences = await updateEsAssetReferences(savedObjectsClient, pkgName, esReferences, { + assetsToAdd: pipelineRefs, }); - if (!installedPkg) throw new Error("integration wasn't found while installing pipelines"); - // remove the current pipeline refs, if any exist, associated with this version before saving new ones so no duplicates occur - await deletePipelineRefs( - savedObjectsClient, - installedPkg.attributes.installed_es, - pkgName, - pkgVersion - ); - await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, pipelineRefs); + const pipelines = dataStreams ? dataStreams.reduce>>((acc, dataStream) => { if (dataStream.ingest_pipeline) { @@ -130,7 +118,8 @@ export const installPipelines = async ( ); } - return await Promise.all(pipelines).then((results) => results.flat()); + await Promise.all(pipelines); + return esReferences; }; export function rewriteIngestPipeline( diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts index e9d693bdbfaa8..7e2b6c121bbab 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts @@ -10,54 +10,38 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/ import { appContextService } from '../../..'; import { ElasticsearchAssetType } from '../../../../types'; import { IngestManagerError } from '../../../../errors'; -import { getInstallation } from '../../packages/get'; -import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common'; import type { EsAssetReference } from '../../../../../common'; +import { updateEsAssetReferences } from '../../packages/install'; export const deletePreviousPipelines = async ( esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract, pkgName: string, - previousPkgVersion: string + previousPkgVersion: string, + esReferences: EsAssetReference[] ) => { const logger = appContextService.getLogger(); - const installation = await getInstallation({ savedObjectsClient, pkgName }); - if (!installation) return; - const installedEsAssets = installation.installed_es; - const installedPipelines = installedEsAssets.filter( + const installedPipelines = esReferences.filter( ({ type, id }) => type === ElasticsearchAssetType.ingestPipeline && id.includes(previousPkgVersion) ); - const deletePipelinePromises = installedPipelines.map(({ type, id }) => { - return deletePipeline(esClient, id); - }); - try { - await Promise.all(deletePipelinePromises); - } catch (e) { - logger.error(e); - } try { - await deletePipelineRefs(savedObjectsClient, installedEsAssets, pkgName, previousPkgVersion); + await Promise.all( + installedPipelines.map(({ type, id }) => { + return deletePipeline(esClient, id); + }) + ); } catch (e) { logger.error(e); } -}; -export const deletePipelineRefs = async ( - savedObjectsClient: SavedObjectsClientContract, - installedEsAssets: EsAssetReference[], - pkgName: string, - pkgVersion: string -) => { - const filteredAssets = installedEsAssets.filter(({ type, id }) => { - if (type !== ElasticsearchAssetType.ingestPipeline) return true; - if (!id.includes(pkgVersion)) return true; - return false; - }); - return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { - installed_es: filteredAssets, + return await updateEsAssetReferences(savedObjectsClient, pkgName, esReferences, { + assetsToRemove: esReferences.filter(({ type, id }) => { + return type === ElasticsearchAssetType.ingestPipeline && id.includes(previousPkgVersion); + }), }); }; + export async function deletePipeline(esClient: ElasticsearchClient, id: string): Promise { // '*' shouldn't ever appear here, but it still would delete all ingest pipelines if (id && id !== '*') { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ml_model/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ml_model/install.ts index 13b3de989e620..630433e18ce39 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ml_model/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ml_model/install.ts @@ -8,13 +8,14 @@ import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server'; import { errors } from '@elastic/elasticsearch'; -import { saveInstalledEsRefs } from '../../packages/install'; import { getPathParts } from '../../archive'; import { ElasticsearchAssetType } from '../../../../../common/types/models'; import type { EsAssetReference, InstallablePackage } from '../../../../../common/types/models'; import { retryTransientEsErrors } from '../retry'; +import { updateEsAssetReferences } from '../../packages/install'; + import { getAsset } from './common'; interface MlModelInstallation { @@ -27,11 +28,11 @@ export const installMlModel = async ( paths: string[], esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract, - logger: Logger + logger: Logger, + esReferences: EsAssetReference[] ) => { const mlModelPath = paths.find((path) => isMlModel(path)); - const installedMlModels: EsAssetReference[] = []; if (mlModelPath !== undefined) { const content = getAsset(mlModelPath).toString('utf-8'); const pathParts = mlModelPath.split('/'); @@ -43,17 +44,22 @@ export const installMlModel = async ( }; // get and save ml model refs before installing ml model - await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, [mlModelRef]); + esReferences = await updateEsAssetReferences( + savedObjectsClient, + installablePackage.name, + esReferences, + { assetsToAdd: [mlModelRef] } + ); const mlModel: MlModelInstallation = { installationName: modelId, content, }; - const result = await handleMlModelInstall({ esClient, logger, mlModel }); - installedMlModels.push(result); + await handleMlModelInstall({ esClient, logger, mlModel }); } - return installedMlModels; + + return esReferences; }; const isMlModel = (path: string) => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts index 48f070434530a..998d0f9fb1ae5 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts @@ -180,15 +180,11 @@ describe('EPM install', () => { packageName: pkg.name, }); - const removeAliases = ( - esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest - ).body; - expect(removeAliases?.template?.aliases).not.toBeDefined(); - const sentTemplate = ( - esClient.indices.putIndexTemplate.mock.calls[1][0] as estypes.IndicesPutIndexTemplateRequest + esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest ).body; expect(sentTemplate).toBeDefined(); + expect(sentTemplate?.template?.aliases).not.toBeDefined(); expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixUnset); expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 6d953835dfe6c..2d2e5b2ffea2a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -16,17 +16,17 @@ import type { RegistryElasticsearch, InstallablePackage, IndexTemplate, - PackageInfo, IndexTemplateMappings, TemplateMapEntry, TemplateMap, + EsAssetReference, } from '../../../../types'; import { loadFieldsFromYaml, processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; import { getAsset, getPathParts } from '../../archive'; -import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install'; +import { updateEsAssetReferences } from '../../packages/install'; import { FLEET_COMPONENT_TEMPLATES, PACKAGE_TEMPLATE_SUFFIX, @@ -36,8 +36,6 @@ import { import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; -import { getPackageInfo } from '../../packages'; - import { generateMappings, generateTemplateName, @@ -54,8 +52,12 @@ export const installTemplates = async ( esClient: ElasticsearchClient, logger: Logger, paths: string[], - savedObjectsClient: SavedObjectsClientContract -): Promise => { + savedObjectsClient: SavedObjectsClientContract, + esReferences: EsAssetReference[] +): Promise<{ + installedTemplates: IndexTemplateEntry[]; + installedEsReferences: EsAssetReference[]; +}> => { // install any pre-built index template assets, // atm, this is only the base package's global index templates // Install component templates first, as they are used by the index templates @@ -63,24 +65,27 @@ export const installTemplates = async ( await installPreBuiltTemplates(paths, esClient, logger); // remove package installation's references to index templates - await removeAssetTypesFromInstalledEs(savedObjectsClient, installablePackage.name, [ - ElasticsearchAssetType.indexTemplate, - ElasticsearchAssetType.componentTemplate, - ]); + esReferences = await updateEsAssetReferences( + savedObjectsClient, + installablePackage.name, + esReferences, + { + assetsToRemove: esReferences.filter( + ({ type }) => + type === ElasticsearchAssetType.indexTemplate || + type === ElasticsearchAssetType.componentTemplate + ), + } + ); + // build templates per data stream from yml files const dataStreams = installablePackage.data_streams; - if (!dataStreams) return []; - - const packageInfo = await getPackageInfo({ - savedObjectsClient, - pkgName: installablePackage.name, - pkgVersion: installablePackage.version, - }); + if (!dataStreams) return { installedTemplates: [], installedEsReferences: esReferences }; const installedTemplatesNested = await Promise.all( dataStreams.map((dataStream) => installTemplateForDataStream({ - pkg: packageInfo, + pkg: installablePackage, esClient, logger, dataStream, @@ -93,13 +98,14 @@ export const installTemplates = async ( const installedIndexTemplateRefs = getAllTemplateRefs(installedTemplates); // add package installation's references to index templates - await saveInstalledEsRefs( + esReferences = await updateEsAssetReferences( savedObjectsClient, installablePackage.name, - installedIndexTemplateRefs + esReferences, + { assetsToAdd: installedIndexTemplateRefs } ); - return installedTemplates; + return { installedTemplates, installedEsReferences: esReferences }; }; const installPreBuiltTemplates = async ( @@ -192,7 +198,7 @@ export async function installTemplateForDataStream({ logger, dataStream, }: { - pkg: PackageInfo; + pkg: InstallablePackage; esClient: ElasticsearchClient; logger: Logger; dataStream: RegistryDataStream; @@ -315,19 +321,20 @@ async function installDataStreamComponentTemplates(params: { await Promise.all( templateEntries.map(async ([name, body]) => { if (isUserSettingsTemplate(name)) { - // look for existing user_settings template - const result = await retryTransientEsErrors( - () => esClient.cluster.getComponentTemplate({ name }, { ignore: [404] }), - { logger } - ); - const hasUserSettingsTemplate = result.component_templates?.length === 1; - if (!hasUserSettingsTemplate) { - // only add if one isn't already present + try { + // Attempt to create custom component templates, ignore if they already exist const { clusterPromise } = putComponentTemplate(esClient, logger, { body, name, + create: true, }); - return clusterPromise; + return await clusterPromise; + } catch (e) { + if (e?.statusCode === 400 && e.body?.error?.reason.includes('already exists')) { + // ignore + } else { + throw e; + } } } else { const { clusterPromise } = putComponentTemplate(esClient, logger, { body, name }); @@ -410,44 +417,6 @@ export async function installTemplate({ }); } - // Datastream now throw an error if the aliases field is present so ensure that we remove that field. - const getTemplateRes = await retryTransientEsErrors( - () => - esClient.indices.getIndexTemplate( - { - name: templateName, - }, - { - ignore: [404], - } - ), - { logger } - ); - - const existingIndexTemplate = getTemplateRes?.index_templates?.[0]; - if ( - existingIndexTemplate && - existingIndexTemplate.name === templateName && - existingIndexTemplate?.index_template?.template?.aliases - ) { - const updateIndexTemplateParams = { - name: templateName, - body: { - ...existingIndexTemplate.index_template, - template: { - ...existingIndexTemplate.index_template.template, - // Remove the aliases field - aliases: undefined, - }, - }, - }; - - await retryTransientEsErrors( - () => esClient.indices.putIndexTemplate(updateIndexTemplateParams, { ignore: [404] }), - { logger } - ); - } - const defaultSettings = buildDefaultSettings({ templateName, packageName, diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts index fea12f4b139c6..ab8f60e172dcb 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -8,7 +8,7 @@ import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server'; import { errors } from '@elastic/elasticsearch'; -import { saveInstalledEsRefs } from '../../packages/install'; +import { updateEsAssetReferences } from '../../packages/install'; import { getPathParts } from '../../archive'; import { ElasticsearchAssetType } from '../../../../../common/types/models'; import type { EsAssetReference, InstallablePackage } from '../../../../../common/types/models'; @@ -18,7 +18,7 @@ import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; -import { deleteTransforms, deleteTransformRefs } from './remove'; +import { deleteTransforms } from './remove'; import { getAsset } from './common'; interface TransformInstallation { @@ -31,12 +31,14 @@ export const installTransform = async ( paths: string[], esClient: ElasticsearchClient, savedObjectsClient: SavedObjectsClientContract, - logger: Logger + logger: Logger, + esReferences?: EsAssetReference[] ) => { const installation = await getInstallation({ savedObjectsClient, pkgName: installablePackage.name, }); + esReferences = esReferences ?? installation?.installed_es ?? []; let previousInstalledTransformEsAssets: EsAssetReference[] = []; if (installation) { previousInstalledTransformEsAssets = installation.installed_es.filter( @@ -71,7 +73,14 @@ export const installTransform = async ( }, []); // get and save transform refs before installing transforms - await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, transformRefs); + esReferences = await updateEsAssetReferences( + savedObjectsClient, + installablePackage.name, + esReferences, + { + assetsToAdd: transformRefs, + } + ); const transforms: TransformInstallation[] = transformPaths.map((path: string) => { const content = JSON.parse(getAsset(path).toString('utf-8')); @@ -95,21 +104,17 @@ export const installTransform = async ( } if (previousInstalledTransformEsAssets.length > 0) { - const currentInstallation = await getInstallation({ + esReferences = await updateEsAssetReferences( savedObjectsClient, - pkgName: installablePackage.name, - }); - - // remove the saved object reference - await deleteTransformRefs( - savedObjectsClient, - currentInstallation?.installed_es || [], installablePackage.name, - previousInstalledTransformEsAssets.map((asset) => asset.id), - installedTransforms.map((installed) => installed.id) + esReferences, + { + assetsToRemove: previousInstalledTransformEsAssets, + } ); } - return installedTransforms; + + return { installedTransforms, esReferences }; }; export const isTransform = (path: string) => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts index 16384b8bfba19..74e49031861c1 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/transform.test.ts @@ -34,8 +34,10 @@ import { appContextService } from '../../../app_context'; import { getESAssetMetadata } from '../meta'; -import { installTransform } from './install'; +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../constants'; + import { getAsset } from './common'; +import { installTransform } from './install'; describe('test transform install', () => { let esClient: ReturnType; @@ -46,6 +48,12 @@ describe('test transform install', () => { (getInstallation as jest.MockedFunction).mockReset(); (getInstallationObject as jest.MockedFunction).mockReset(); savedObjectsClient = savedObjectsClientMock.create(); + savedObjectsClient.update.mockImplementation(async (type, id, attributes) => ({ + type: PACKAGES_SAVED_OBJECT_TYPE, + id: 'endpoint', + attributes, + references: [], + })); }); afterEach(() => { @@ -158,7 +166,8 @@ describe('test transform install', () => { ], esClient, savedObjectsClient, - loggerMock.create() + loggerMock.create(), + previousInstallation.installed_es ); expect(esClient.transform.getTransform.mock.calls).toEqual([ @@ -255,6 +264,9 @@ describe('test transform install', () => { }, ], }, + { + refresh: false, + }, ], [ 'epm-packages', @@ -266,15 +278,18 @@ describe('test transform install', () => { type: 'ingest_pipeline', }, { - id: 'endpoint.metadata_current-default-0.16.0-dev.0', + id: 'endpoint.metadata-default-0.16.0-dev.0', type: 'transform', }, { - id: 'endpoint.metadata-default-0.16.0-dev.0', + id: 'endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform', }, ], }, + { + refresh: false, + }, ], ]); }); @@ -331,7 +346,8 @@ describe('test transform install', () => { ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'], esClient, savedObjectsClient, - loggerMock.create() + loggerMock.create(), + previousInstallation.installed_es ); const meta = getESAssetMetadata({ packageName: 'endpoint' }); @@ -363,6 +379,9 @@ describe('test transform install', () => { { id: 'endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform' }, ], }, + { + refresh: false, + }, ], ]); }); @@ -443,7 +462,8 @@ describe('test transform install', () => { [], esClient, savedObjectsClient, - loggerMock.create() + loggerMock.create(), + previousInstallation.installed_es ); expect(esClient.transform.getTransform.mock.calls).toEqual([ @@ -492,6 +512,9 @@ describe('test transform install', () => { { installed_es: [], }, + { + refresh: false, + }, ], ]); }); @@ -559,7 +582,8 @@ describe('test transform install', () => { ['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'], esClient, savedObjectsClient, - loggerMock.create() + loggerMock.create(), + previousInstallation.installed_es ); const meta = getESAssetMetadata({ packageName: 'endpoint' }); @@ -586,6 +610,9 @@ describe('test transform install', () => { { id: 'endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform' }, ], }, + { + refresh: false, + }, ], ]); }); diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.ts index f1ad96504594e..3f1a8d8b2b7ba 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/field.ts +++ b/x-pack/plugins/fleet/server/services/epm/fields/field.ts @@ -262,7 +262,7 @@ const isFields = (path: string) => { */ export const loadFieldsFromYaml = async ( - pkg: PackageInfo, + pkg: Pick, datasetName?: string ): Promise => { // Fetch all field definition files diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts index ce8d7e7be2bb9..1462cd61c4bd3 100644 --- a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts @@ -21,9 +21,11 @@ import { partition } from 'lodash'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common'; import { getAsset, getPathParts } from '../../archive'; import { KibanaAssetType, KibanaSavedObjectType } from '../../../../types'; -import type { AssetType, AssetReference, AssetParts } from '../../../../types'; +import type { AssetType, AssetReference, AssetParts, Installation } from '../../../../types'; import { savedObjectTypes } from '../../packages'; import { indexPatternTypes, getIndexPatternSavedObjects } from '../index_pattern/install'; +import { saveKibanaAssetsRefs } from '../../packages/install'; +import { deleteKibanaSavedObjectsAssets } from '../../packages/remove'; type SavedObjectsImporterContract = Pick; const formatImportErrorsForLog = (errors: SavedObjectsImportFailure[]) => @@ -121,6 +123,41 @@ export async function installKibanaAssets(options: { return installedAssets; } + +export async function installKibanaAssetsAndReferences({ + savedObjectsClient, + savedObjectsImporter, + logger, + pkgName, + paths, + installedPkg, +}: { + savedObjectsClient: SavedObjectsClientContract; + savedObjectsImporter: Pick; + logger: Logger; + pkgName: string; + paths: string[]; + installedPkg?: SavedObject; +}) { + const kibanaAssets = await getKibanaAssets(paths); + if (installedPkg) await deleteKibanaSavedObjectsAssets({ savedObjectsClient, installedPkg }); + // save new kibana refs before installing the assets + const installedKibanaAssetsRefs = await saveKibanaAssetsRefs( + savedObjectsClient, + pkgName, + kibanaAssets + ); + + await installKibanaAssets({ + logger, + savedObjectsImporter, + pkgName, + kibanaAssets, + }); + + return installedKibanaAssetsRefs; +} + export const deleteKibanaInstalledRefs = async ( savedObjectsClient: SavedObjectsClientContract, pkgName: string, diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts index 31bf9e47a4ae0..782af2860d2e3 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts @@ -50,6 +50,7 @@ function getTest( spy: jest.SpyInstance; spyArgs: any[]; spyResponse: any; + expectedReturnValue: any; }; switch (testKey) { @@ -65,6 +66,7 @@ function getTest( }, ], spyResponse: { name: 'getInstallation test' }, + expectedReturnValue: { name: 'getInstallation test' }, }; break; case testKeys[1]: @@ -82,6 +84,7 @@ function getTest( }, ], spyResponse: { name: 'ensureInstalledPackage test' }, + expectedReturnValue: { name: 'ensureInstalledPackage test' }, }; break; case testKeys[2]: @@ -91,6 +94,7 @@ function getTest( spy: jest.spyOn(epmRegistry, 'fetchFindLatestPackageOrThrow'), spyArgs: ['package name'], spyResponse: { name: 'fetchFindLatestPackage test' }, + expectedReturnValue: { name: 'fetchFindLatestPackage test' }, }; break; case testKeys[3]: @@ -103,6 +107,10 @@ function getTest( packageInfo: { name: 'getRegistryPackage test' }, paths: ['/some/test/path'], }, + expectedReturnValue: { + packageInfo: { name: 'getRegistryPackage test' }, + paths: ['/some/test/path'], + }, }; break; case testKeys[4]: @@ -122,7 +130,14 @@ function getTest( args: [pkg, paths], spy: jest.spyOn(epmTransformsInstall, 'installTransform'), spyArgs: [pkg, paths, mocks.esClient, mocks.soClient, mocks.logger], - spyResponse: [ + spyResponse: { + installedTransforms: [ + { + name: 'package name', + }, + ], + }, + expectedReturnValue: [ { name: 'package name', }, @@ -176,10 +191,13 @@ describe('PackageService', () => { soClient: mockSoClient, logger: mockLogger, }; - const { method, args, spy, spyArgs, spyResponse } = getTest(mockClients, testKey); + const { method, args, spy, spyArgs, spyResponse, expectedReturnValue } = getTest( + mockClients, + testKey + ); spy.mockResolvedValue(spyResponse); - await expect(method(...args)).resolves.toEqual(spyResponse); + await expect(method(...args)).resolves.toEqual(expectedReturnValue); expect(spy).toHaveBeenCalledWith(...spyArgs); }); }); @@ -193,10 +211,13 @@ describe('PackageService', () => { soClient: mockSoClient, logger: mockLogger, }; - const { method, args, spy, spyArgs, spyResponse } = getTest(mockClients, testKey); + const { method, args, spy, spyArgs, spyResponse, expectedReturnValue } = getTest( + mockClients, + testKey + ); spy.mockResolvedValue(spyResponse); - await expect(method(...args)).resolves.toEqual(spyResponse); + await expect(method(...args)).resolves.toEqual(expectedReturnValue); expect(spy).toHaveBeenCalledWith(...spyArgs); }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index 573ca3508e947..e16d4954f0b9d 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -146,14 +146,15 @@ class PackageClientImpl implements PackageClient { return installedAssets; } - #reinstallTransforms(packageInfo: InstallablePackage, paths: string[]) { - return installTransform( + async #reinstallTransforms(packageInfo: InstallablePackage, paths: string[]) { + const { installedTransforms } = await installTransform( packageInfo, paths, this.internalEsClient, this.internalSoClient, this.logger ); + return installedTransforms; } #runPreflight() { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts index c0e4404345902..db9803ea70f3a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts @@ -21,16 +21,15 @@ jest.mock('./install'); jest.mock('./get'); import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; -import { installKibanaAssets } from '../kibana/assets/install'; +import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; import { _installPackage } from './_install_package'; const mockedUpdateCurrentWriteIndices = updateCurrentWriteIndices as jest.MockedFunction< typeof updateCurrentWriteIndices >; -const mockedGetKibanaAssets = installKibanaAssets as jest.MockedFunction< - typeof installKibanaAssets ->; +const mockedInstallKibanaAssetsAndReferences = + installKibanaAssetsAndReferences as jest.MockedFunction; function sleep(millis: number) { return new Promise((resolve) => setTimeout(resolve, millis)); @@ -50,7 +49,7 @@ describe('_installPackage', () => { }); it('handles errors from installKibanaAssets', async () => { // force errors from this function - mockedGetKibanaAssets.mockImplementation(async () => { + mockedInstallKibanaAssetsAndReferences.mockImplementation(async () => { throw new Error('mocked async error A: should be caught'); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 796269eee38b1..24c324e6b7cd0 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -29,9 +29,8 @@ import { isTopLevelPipeline, deletePreviousPipelines, } from '../elasticsearch/ingest_pipeline'; -import { getAllTemplateRefs } from '../elasticsearch/template/install'; import { installILMPolicy } from '../elasticsearch/ilm/install'; -import { installKibanaAssets, getKibanaAssets } from '../kibana/assets/install'; +import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; import { installTransform } from '../elasticsearch/transform/install'; import { installMlModel } from '../elasticsearch/ml_model'; @@ -40,8 +39,7 @@ import { saveArchiveEntries } from '../archive/storage'; import { ConcurrentInstallOperationError } from '../../../errors'; import { packagePolicyService } from '../..'; -import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install'; -import { deleteKibanaSavedObjectsAssets } from './remove'; +import { createInstallation } from './install'; import { withPackageSpan } from './utils'; // this is only exported for testing @@ -106,47 +104,59 @@ export async function _installPackage({ }); } - const installedKibanaAssetsRefs = await withPackageSpan('Install Kibana assets', async () => { - const kibanaAssets = await getKibanaAssets(paths); - if (installedPkg) await deleteKibanaSavedObjectsAssets({ savedObjectsClient, installedPkg }); - // save new kibana refs before installing the assets - const assetRefs = await saveKibanaAssetsRefs(savedObjectsClient, pkgName, kibanaAssets); - - await installKibanaAssets({ - logger, + const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => + installKibanaAssetsAndReferences({ + savedObjectsClient, savedObjectsImporter, pkgName, - kibanaAssets, - }); + paths, + installedPkg, + logger, + }) + ); + // Necessary to avoid async promise rejection warning + // See https://stackoverflow.com/questions/40920179/should-i-refrain-from-handling-promise-rejection-asynchronously + kibanaAssetPromise.catch(() => {}); - return assetRefs; - }); + // Use a shared array that is updated by each operation. This allows each operation to accurately update the + // installation object with it's references without requiring a refresh of the SO index on each update (faster). + let esReferences = installedPkg?.attributes.installed_es ?? []; // the rest of the installation must happen in sequential order // currently only the base package has an ILM policy // at some point ILM policies can be installed/modified // per data stream and we should then save them - await withPackageSpan('Install ILM policies', () => - installILMPolicy(packageInfo, paths, esClient, logger) + esReferences = await withPackageSpan('Install ILM policies', () => + installILMPolicy(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) ); - const installedDataStreamIlm = await withPackageSpan('Install Data Stream ILM policies', () => - installIlmForDataStream(packageInfo, paths, esClient, savedObjectsClient, logger) - ); + ({ esReferences } = await withPackageSpan('Install Data Stream ILM policies', () => + installIlmForDataStream( + packageInfo, + paths, + esClient, + savedObjectsClient, + logger, + esReferences + ) + )); // installs ml models - const installedMlModel = await withPackageSpan('Install ML models', () => - installMlModel(packageInfo, paths, esClient, savedObjectsClient, logger) + esReferences = await withPackageSpan('Install ML models', () => + installMlModel(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) ); // installs versionized pipelines without removing currently installed ones - const installedPipelines = await withPackageSpan('Install ingest pipelines', () => - installPipelines(packageInfo, paths, esClient, savedObjectsClient, logger) + esReferences = await withPackageSpan('Install ingest pipelines', () => + installPipelines(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) ); + // install or update the templates referencing the newly installed pipelines - const installedTemplates = await withPackageSpan('Install index templates', () => - installTemplates(packageInfo, esClient, logger, paths, savedObjectsClient) - ); + const { installedTemplates, installedEsReferences: esReferencesAfterTemplates } = + await withPackageSpan('Install index templates', () => + installTemplates(packageInfo, esClient, logger, paths, savedObjectsClient, esReferences) + ); + esReferences = esReferencesAfterTemplates; try { await removeLegacyTemplates({ packageInfo, esClient, logger }); @@ -159,9 +169,9 @@ export async function _installPackage({ updateCurrentWriteIndices(esClient, logger, installedTemplates) ); - const installedTransforms = await withPackageSpan('Install transforms', () => - installTransform(packageInfo, paths, esClient, savedObjectsClient, logger) - ); + ({ esReferences } = await withPackageSpan('Install transforms', () => + installTransform(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences) + )); // If this is an update or retrying an update, delete the previous version's pipelines // Top-level pipeline assets will not be removed on upgrade as of ml model package addition which requires previous @@ -171,28 +181,30 @@ export async function _installPackage({ (installType === 'update' || installType === 'reupdate') && installedPkg ) { - await withPackageSpan('Delete previous ingest pipelines', () => + esReferences = await withPackageSpan('Delete previous ingest pipelines', () => deletePreviousPipelines( esClient, savedObjectsClient, pkgName, - installedPkg.attributes.version + installedPkg!.attributes.version, + esReferences ) ); } // pipelines from a different version may have installed during a failed update if (installType === 'rollback' && installedPkg) { - await await withPackageSpan('Delete previous ingest pipelines', () => + esReferences = await withPackageSpan('Delete previous ingest pipelines', () => deletePreviousPipelines( esClient, savedObjectsClient, pkgName, - installedPkg.attributes.install_version + installedPkg!.attributes.install_version, + esReferences ) ); } - const installedTemplateRefs = getAllTemplateRefs(installedTemplates); + const installedKibanaAssetsRefs = await kibanaAssetPromise; const packageAssetResults = await withPackageSpan('Update archive entries', () => saveArchiveEntries({ savedObjectsClient, @@ -208,11 +220,9 @@ export async function _installPackage({ }) ); - // update to newly installed version when all assets are successfully installed - if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion); - const updatedPackage = await withPackageSpan('Update install status', () => savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { + version: pkgVersion, install_version: pkgVersion, install_status: 'installed', package_assets: packageAssetRefs, @@ -233,14 +243,7 @@ export async function _installPackage({ }); } - return [ - ...installedKibanaAssetsRefs, - ...installedPipelines, - ...installedDataStreamIlm, - ...installedTemplateRefs, - ...installedTransforms, - ...installedMlModel, - ]; + return [...installedKibanaAssetsRefs, ...esReferences]; } catch (err) { if (savedObjectsClient.errors.isConflictError(err)) { throw new ConcurrentInstallOperationError( diff --git a/x-pack/plugins/fleet/server/services/epm/packages/assets.ts b/x-pack/plugins/fleet/server/services/epm/packages/assets.ts index c939ce093a65c..0621d05d21497 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/assets.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/assets.ts @@ -17,7 +17,7 @@ import type { ArchiveEntry } from '../archive'; // and different package and version structure export function getAssets( - packageInfo: PackageInfo, + packageInfo: Pick, filter = (path: string): boolean => true, datasetName?: string ): string[] { @@ -52,7 +52,7 @@ export function getAssets( // ASK: Does getAssetsData need an installSource now? // if so, should it be an Installation vs InstallablePackage or add another argument? export async function getAssetsData( - packageInfo: PackageInfo, + packageInfo: Pick, filter = (path: string): boolean => true, datasetName?: string ): Promise { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 9ae549982399c..c7fc01c89eb06 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -17,6 +17,8 @@ import type { import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; +import pRetry from 'p-retry'; + import { generateESIndexPatterns } from '../elasticsearch/template/template'; import type { BulkInstallPackageInfo, @@ -29,13 +31,7 @@ import { IngestManagerError, PackageOutdatedError } from '../../../errors'; import { PACKAGES_SAVED_OBJECT_TYPE, MAX_TIME_COMPLETE_INSTALL } from '../../../constants'; import type { KibanaAssetType } from '../../../types'; import { licenseService } from '../..'; -import type { - Installation, - AssetType, - EsAssetReference, - InstallType, - InstallResult, -} from '../../../types'; +import type { Installation, EsAssetReference, InstallType, InstallResult } from '../../../types'; import { appContextService } from '../../app_context'; import * as Registry from '../registry'; import { @@ -271,10 +267,13 @@ async function installPackageFromRegistry({ installType, }); - // get latest package version - const latestPackage = await Registry.fetchFindLatestPackageOrThrow(pkgName, { - ignoreConstraints, - }); + // get latest package version and requested version in parallel for performance + const [latestPackage, { paths, packageInfo }] = await Promise.all([ + Registry.fetchFindLatestPackageOrThrow(pkgName, { + ignoreConstraints, + }), + Registry.getRegistryPackage(pkgName, pkgVersion), + ]); // let the user install if using the force flag or needing to reinstall or install a previous version due to failed update const installOutOfDateVersionOk = @@ -319,9 +318,6 @@ async function installPackageFromRegistry({ ); } - // get package info - const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion); - if (!licenseService.hasAtLeast(packageInfo.license || 'basic')) { const err = new Error(`Requires ${packageInfo.license} license`); sendEvent({ @@ -632,22 +628,60 @@ export const saveKibanaAssetsRefs = async ( kibanaAssets: Record ) => { const assetRefs = Object.values(kibanaAssets).flat().map(toAssetReference); - await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { - installed_kibana: assetRefs, - }); + // Because Kibana assets are installed in parallel with ES assets with refresh: false, we almost always run into an + // issue that causes a conflict error due to this issue: https://github.com/elastic/kibana/issues/126240. This is safe + // to retry constantly until it succeeds to optimize this critical user journey path as much as possible. + pRetry( + () => + savedObjectsClient.update( + PACKAGES_SAVED_OBJECT_TYPE, + pkgName, + { + installed_kibana: assetRefs, + }, + { refresh: false } + ), + { retries: 20 } // Use a number of retries higher than the number of es asset update operations + ); + return assetRefs; }; -export const saveInstalledEsRefs = async ( +/** + * Utility function for updating the installed_es field of a package + */ +export const updateEsAssetReferences = async ( savedObjectsClient: SavedObjectsClientContract, pkgName: string, - installedAssets: EsAssetReference[] -) => { - const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName }); - const installedAssetsToSave = installedPkg?.attributes.installed_es.concat(installedAssets); + currentAssets: EsAssetReference[], + { + assetsToAdd = [], + assetsToRemove = [], + refresh = false, + }: { + assetsToAdd?: EsAssetReference[]; + assetsToRemove?: EsAssetReference[]; + /** + * Whether or not the update should force a refresh on the SO index. + * Defaults to `false` for faster updates, should only be `wait_for` if the update needs to be queried back from ES + * immediately. + */ + refresh?: 'wait_for' | false; + } +): Promise => { + const withAssetsRemoved = currentAssets.filter(({ type, id }) => { + if ( + assetsToRemove.some( + ({ type: removeType, id: removeId }) => removeType === type && removeId === id + ) + ) { + return false; + } + return true; + }); const deduplicatedAssets = - installedAssetsToSave?.reduce((acc, currentAsset) => { + [...withAssetsRemoved, ...assetsToAdd].reduce((acc, currentAsset) => { const foundAsset = acc.find((asset: EsAssetReference) => asset.id === currentAsset.id); if (!foundAsset) { return acc.concat([currentAsset]); @@ -656,27 +690,30 @@ export const saveInstalledEsRefs = async ( } }, [] as EsAssetReference[]) || []; - await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { - installed_es: deduplicatedAssets, - }); - return installedAssets; -}; - -export const removeAssetTypesFromInstalledEs = async ( - savedObjectsClient: SavedObjectsClientContract, - pkgName: string, - assetTypes: AssetType[] -) => { - const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName }); - const installedAssets = installedPkg?.attributes.installed_es; - if (!installedAssets?.length) return; - const installedAssetsToSave = installedAssets?.filter( - (asset) => !assetTypes.includes(asset.type) - ); + const { + attributes: { installed_es: updatedAssets }, + } = + // Because Kibana assets are installed in parallel with ES assets with refresh: false, we almost always run into an + // issue that causes a conflict error due to this issue: https://github.com/elastic/kibana/issues/126240. This is safe + // to retry constantly until it succeeds to optimize this critical user journey path as much as possible. + await pRetry( + () => + savedObjectsClient.update( + PACKAGES_SAVED_OBJECT_TYPE, + pkgName, + { + installed_es: deduplicatedAssets, + }, + { + refresh, + } + ), + // Use a lower number of retries for ES assets since they're installed in serial and can only conflict with + // the single Kibana update call. + { retries: 5 } + ); - return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { - installed_es: installedAssetsToSave, - }); + return updatedAssets ?? []; }; export async function ensurePackagesCompletedInstall( diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index 7edf5b6020be8..95e65acfebef6 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -130,6 +130,8 @@ function deleteESAssets( return deleteTransforms(esClient, [id]); } else if (assetType === ElasticsearchAssetType.dataStreamIlmPolicy) { return deleteIlms(esClient, [id]); + } else if (assetType === ElasticsearchAssetType.ilmPolicy) { + return deleteIlms(esClient, [id]); } else if (assetType === ElasticsearchAssetType.mlModel) { return deleteMlModel(esClient, [id]); } diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 2ae531f63379d..1074e975d3f6f 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -152,7 +152,8 @@ export async function fetchFindLatestPackageOrUndefined( export async function fetchInfo(pkgName: string, pkgVersion: string): Promise { const registryUrl = getRegistryUrl(); try { - const res = await fetchUrl(`${registryUrl}/package/${pkgName}/${pkgVersion}`).then(JSON.parse); + // Trailing slash avoids 301 redirect / extra hop + const res = await fetchUrl(`${registryUrl}/package/${pkgName}/${pkgVersion}/`).then(JSON.parse); return res; } catch (err) { diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 8f2b3effd94ed..16f8fc04aa92f 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -569,6 +569,10 @@ const expectAssetsInstalled = ({ id: 'metrics-all_assets.test_metrics-all_assets', type: 'data_stream_ilm_policy', }, + { + id: 'all_assets', + type: 'ilm_policy', + }, { id: 'logs-all_assets.test_logs', type: 'index_template', diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index b73ca9537990c..9758107cee83d 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -403,13 +403,17 @@ export default function (providerContext: FtrProviderContext) { ], installed_es: [ { - id: 'logs-all_assets.test_logs-all_assets', - type: 'data_stream_ilm_policy', + id: 'all_assets', + type: 'ilm_policy', }, { id: 'default', type: 'ml_model', }, + { + id: 'logs-all_assets.test_logs-all_assets', + type: 'data_stream_ilm_policy', + }, { id: 'logs-all_assets.test_logs-0.2.0', type: 'ingest_pipeline',