Optimize package installation performance, phase 2

x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts

@@ -12,7 +12,6 @@ import { ElasticsearchAssetType } from '../../../../types';
 import type { EsAssetReference, RegistryDataStream, InstallablePackage } from '../../../../types';
 import { getAsset, getPathParts } from '../../archive';
 import type { ArchiveEntry } from '../../archive';
-import { updateEsAssetReferences } from '../../packages/install';
 import {
   FLEET_FINAL_PIPELINE_CONTENT,
   FLEET_FINAL_PIPELINE_ID,
@@ -48,11 +47,12 @@ export const installPipelines = async (
   // it can be created pointing to the new template, without removing the old one and effecting data
   // so do not remove the currently installed pipelines here
   const dataStreams = installablePackage.data_streams;
-  const { name: pkgName, version: pkgVersion } = installablePackage;
+  const { version: pkgVersion } = installablePackage;
   const pipelinePaths = paths.filter((path) => isPipeline(path));
   const topLevelPipelinePaths = paths.filter((path) => isTopLevelPipeline(path));
 
-  if (!dataStreams?.length && topLevelPipelinePaths.length === 0) return [];
+  if (!dataStreams?.length && topLevelPipelinePaths.length === 0)
+    return { assetsToAdd: [], assetsToRemove: [] };
 
   // get and save pipeline refs before installing pipelines
   let pipelineRefs = dataStreams
@@ -85,10 +85,6 @@ export const installPipelines = async (
 
   pipelineRefs = [...pipelineRefs, ...topLevelPipelineRefs];
 
-  esReferences = await updateEsAssetReferences(savedObjectsClient, pkgName, esReferences, {
-    assetsToAdd: pipelineRefs,
-  });
-
   const pipelines = dataStreams
     ? dataStreams.reduce<Array<Promise<EsAssetReference[]>>>((acc, dataStream) => {
         if (dataStream.ingest_pipeline) {
@@ -119,7 +115,9 @@ export const installPipelines = async (
   }
 
   await Promise.all(pipelines);
-  return esReferences;
+  return {
+    assetsToAdd: pipelineRefs,
+  };
 };
 
 export function rewriteIngestPipeline(

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -7,7 +7,7 @@
 
 import { merge } from 'lodash';
 import Boom from '@hapi/boom';
-import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server';
+import type { ElasticsearchClient, Logger } from '@kbn/core/server';
 
 import { ElasticsearchAssetType } from '../../../../types';
 import type {
@@ -26,7 +26,6 @@ import { loadFieldsFromYaml, processFields } from '../../fields/field';
 import type { Field } from '../../fields/field';
 import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
 import { getAsset, getPathParts } from '../../archive';
-import { updateEsAssetReferences } from '../../packages/install';
 import {
   FLEET_COMPONENT_TEMPLATES,
   PACKAGE_TEMPLATE_SUFFIX,
@@ -52,11 +51,13 @@ export const installTemplates = async (
   esClient: ElasticsearchClient,
   logger: Logger,
   paths: string[],
-  savedObjectsClient: SavedObjectsClientContract,
   esReferences: EsAssetReference[]
 ): Promise<{
   installedTemplates: IndexTemplateEntry[];
-  installedEsReferences: EsAssetReference[];
+  esReferences: {
+    assetsToAdd: EsAssetReference[];
+    assetsToRemove: EsAssetReference[];
+  };
 }> => {
   // install any pre-built index template assets,
   // atm, this is only the base package's global index templates
@@ -65,22 +66,16 @@ export const installTemplates = async (
   await installPreBuiltTemplates(paths, esClient, logger);
 
   // remove package installation's references to index templates
-  esReferences = await updateEsAssetReferences(
-    savedObjectsClient,
-    installablePackage.name,
-    esReferences,
-    {
-      assetsToRemove: esReferences.filter(
-        ({ type }) =>
-          type === ElasticsearchAssetType.indexTemplate ||
-          type === ElasticsearchAssetType.componentTemplate
-      ),
-    }
+  const assetsToRemove = esReferences.filter(
+    ({ type }) =>
+      type === ElasticsearchAssetType.indexTemplate ||
+      type === ElasticsearchAssetType.componentTemplate
   );
 
   // build templates per data stream from yml files
   const dataStreams = installablePackage.data_streams;
-  if (!dataStreams) return { installedTemplates: [], installedEsReferences: esReferences };
+  if (!dataStreams)
+    return { installedTemplates: [], esReferences: { assetsToAdd: [], assetsToRemove } };
 
   const installedTemplatesNested = await Promise.all(
     dataStreams.map((dataStream) =>
@@ -97,15 +92,14 @@ export const installTemplates = async (
   // get template refs to save
   const installedIndexTemplateRefs = getAllTemplateRefs(installedTemplates);
 
-  // add package installation's references to index templates
-  esReferences = await updateEsAssetReferences(
-    savedObjectsClient,
-    installablePackage.name,
-    esReferences,
-    { assetsToAdd: installedIndexTemplateRefs }
-  );
-
-  return { installedTemplates, installedEsReferences: esReferences };
+  return {
+    installedTemplates,
+    esReferences: {
+      // add package installation's references to index templates
+      assetsToAdd: installedIndexTemplateRefs,
+      assetsToRemove,
+    },
+  };
 };
 
 const installPreBuiltTemplates = async (

x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts

@@ -267,6 +267,7 @@ export async function installKibanaSavedObjects({
         overwrite: true,
         readStream: createListStream(toBeSavedObjects),
         createNewCopies: false,
+        refresh: false,
       })
     );
 

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -39,7 +39,7 @@ import { saveArchiveEntries } from '../archive/storage';
 import { ConcurrentInstallOperationError } from '../../../errors';
 import { packagePolicyService } from '../..';
 
-import { createInstallation } from './install';
+import { createInstallation, updateEsAssetReferences } from './install';
 import { withPackageSpan } from './utils';
 
 // this is only exported for testing
@@ -146,17 +146,31 @@ export async function _installPackage({
       installMlModel(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences)
     );
 
-    // installs versionized pipelines without removing currently installed ones
-    esReferences = await withPackageSpan('Install ingest pipelines', () =>
-      installPipelines(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences)
-    );
+    // Install index templates and ingest pipelines in parallel since they typically take the longest
+    const [ingestRefs, templateResults] = await Promise.all([
+      // installs versionized pipelines without removing currently installed ones
+      withPackageSpan('Install ingest pipelines', () =>
+        installPipelines(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences)
+      ),
+      withPackageSpan('Install index templates', () =>
+        installTemplates(packageInfo, esClient, logger, paths, esReferences)
+      ),
+    ]);
 
-    // install or update the templates referencing the newly installed pipelines
-    const { installedTemplates, installedEsReferences: esReferencesAfterTemplates } =
-      await withPackageSpan('Install index templates', () =>
-        installTemplates(packageInfo, esClient, logger, paths, savedObjectsClient, esReferences)
-      );
-    esReferences = esReferencesAfterTemplates;
+    // Update the references for the templates and ingest pipelines together. Need to be done togther to avoid race
+    // conditions on updating the installed_es field at the same time
+    esReferences = await updateEsAssetReferences(
+      savedObjectsClient,
+      packageInfo.name,
+      esReferences,
+      {
+        assetsToRemove: [
+          ...ingestRefs.assetsToRemove,
+          ...templateResults.esReferences.assetsToRemove,
+        ],
+        assetsToAdd: [...ingestRefs.assetsToAdd, ...templateResults.esReferences.assetsToAdd],
+      }
+    );
 
     try {
       await removeLegacyTemplates({ packageInfo, esClient, logger });
@@ -166,7 +180,7 @@ export async function _installPackage({
 
     // update current backing indices of each data stream
     await withPackageSpan('Update write indices', () =>
-      updateCurrentWriteIndices(esClient, logger, installedTemplates)
+      updateCurrentWriteIndices(esClient, logger, templateResults.installedTemplates)
     );
 
     ({ esReferences } = await withPackageSpan('Install transforms', () =>

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -616,7 +616,7 @@ export async function createInstallation(options: {
       install_source: installSource,
       keep_policies_up_to_date: defaultKeepPoliciesUpToDate,
     },
-    { id: pkgName, overwrite: true }
+    { id: pkgName, overwrite: true, refresh: false }
   );
 
   return created;