Make creation of preconfigured agent policy functional

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -107,7 +107,7 @@ export type InstallablePackage = RegistryPackage | ArchivePackage;
 
 export type ArchivePackage = PackageSpecManifest &
   // should an uploaded package be able to specify `internal`?
-  Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal'>;
+  Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal' | 'elasticsearch'>;
 
 export type RegistryPackage = PackageSpecManifest &
   Partial<RegistryOverridesToOptional> &

x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts

@@ -31,10 +31,11 @@ import { getMonitoringPermissions } from './monitoring_permissions';
 export async function getFullAgentPolicy(
   soClient: SavedObjectsClientContract,
   id: string,
-  options?: { standalone: boolean }
+  options: Partial<{ standalone: boolean; useBundledPackages: boolean }> = {}
 ): Promise<FullAgentPolicy | null> {
   let agentPolicy;
-  const standalone = options?.standalone;
+  const standalone = options.standalone ?? false;
+  const useBundledPackages = options.useBundledPackages ?? false;
 
   try {
     agentPolicy = await agentPolicyService.get(soClient, id);
@@ -111,7 +112,9 @@ export async function getFullAgentPolicy(
   };
 
   const dataPermissions =
-    (await storedPackagePoliciesToAgentPermissions(soClient, agentPolicy.package_policies)) || {};
+    (await storedPackagePoliciesToAgentPermissions(soClient, agentPolicy.package_policies, {
+      useBundledPackages,
+    })) || {};
 
   dataPermissions._elastic_agent_checks = {
     cluster: DEFAULT_CLUSTER_PERMISSIONS,

x-pack/plugins/fleet/server/services/agent_policy.ts

@@ -602,7 +602,11 @@ class AgentPolicyService {
     };
   }
 
-  public async deployPolicy(soClient: SavedObjectsClientContract, agentPolicyId: string) {
+  public async deployPolicy(
+    soClient: SavedObjectsClientContract,
+    agentPolicyId: string,
+    useBundledPackages: boolean = false
+  ) {
     // Use internal ES client so we have permissions to write to .fleet* indices
     const esClient = appContextService.getInternalUserESClient();
     const defaultOutputId = await outputService.getDefaultDataOutputId(soClient);
@@ -612,7 +616,9 @@ class AgentPolicyService {
     }
 
     const policy = await agentPolicyService.get(soClient, agentPolicyId);
-    const fullPolicy = await agentPolicyService.getFullAgentPolicy(soClient, agentPolicyId);
+    const fullPolicy = await agentPolicyService.getFullAgentPolicy(soClient, agentPolicyId, {
+      useBundledPackages,
+    });
     if (!policy || !fullPolicy || !fullPolicy.revision) {
       return;
     }
@@ -715,7 +721,7 @@ class AgentPolicyService {
   public async getFullAgentPolicy(
     soClient: SavedObjectsClientContract,
     id: string,
-    options?: { standalone: boolean }
+    options: Partial<{ standalone: boolean; useBundledPackages: boolean }> = {}
   ): Promise<FullAgentPolicy | null> {
     return getFullAgentPolicy(soClient, id, options);
   }
@@ -733,12 +739,14 @@ export async function addPackageToAgentPolicy(
   packagePolicyId?: string | number,
   packagePolicyDescription?: string,
   transformPackagePolicy?: (p: NewPackagePolicy) => NewPackagePolicy,
-  bumpAgentPolicyRevison = false
+  bumpAgentPolicyRevison = false,
+  useBundledPackages = false
 ) {
   const packageInfo = await getPackageInfo({
     savedObjectsClient: soClient,
     pkgName: packageToInstall.name,
     pkgVersion: packageToInstall.version,
+    useBundledPackages,
   });
 
   const basePackagePolicy = packageToPackagePolicy(
@@ -767,5 +775,6 @@ export async function addPackageToAgentPolicy(
     skipUniqueNameVerification: true,
     overwrite: true,
     force: true, // To add package to managed policy we need the force flag
+    useBundledPackages,
   });
 }

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -21,14 +21,15 @@ import type {
   GetCategoriesRequest,
 } from '../../../../common/types';
 import type { Installation, PackageInfo } from '../../../types';
-import { IngestManagerError } from '../../../errors';
+import { BundledPackageNotFoundError, IngestManagerError } from '../../../errors';
 import { appContextService } from '../../';
 import * as Registry from '../registry';
 import { getEsPackage } from '../archive/storage';
 import { getArchivePackage } from '../archive';
 import { normalizeKuery } from '../../saved_object';
 
 import { createInstallableFrom } from './index';
+import { getBundledPackages } from './get_bundled_packages';
 
 export type { SearchParams } from '../registry';
 export { getFile } from '../registry';
@@ -102,11 +103,36 @@ export async function getPackageInfo(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;
   pkgVersion: string;
+  useBundledPackages?: boolean;
 }): Promise<PackageInfo> {
-  const { savedObjectsClient, pkgName, pkgVersion } = options;
+  const { savedObjectsClient, pkgName, pkgVersion, useBundledPackages = false } = options;
   const [savedObject, latestPackage] = await Promise.all([
     getInstallationObject({ savedObjectsClient, pkgName }),
-    Registry.fetchFindLatestPackage(pkgName),
+    new Promise<{ name: string; version: string }>((resolve, reject) => {
+      if (!useBundledPackages) {
+        resolve(Registry.fetchFindLatestPackage(pkgName));
+      }
+
+      return Registry.fetchFindLatestPackage(pkgName).catch(async (error) => {
+        const bundledPackages = await getBundledPackages();
+        const bundledPackage = bundledPackages.find(
+          (b) => b.name === pkgName && b.version === pkgVersion
+        );
+
+        if (!bundledPackage) {
+          return reject(
+            new BundledPackageNotFoundError(
+              `No bundled package found with key ${pkgName}-${pkgVersion}`
+            )
+          );
+        }
+
+        resolve({
+          name: bundledPackage.name,
+          version: bundledPackage.version,
+        });
+      });
+    }),
   ]);
 
   // If no package version is provided, use the installed version in the response

x-pack/plugins/fleet/server/services/managed_package_policies.ts

@@ -31,7 +31,8 @@ export interface UpgradeManagedPackagePoliciesResult {
 export const upgradeManagedPackagePolicies = async (
   soClient: SavedObjectsClientContract,
   esClient: ElasticsearchClient,
-  packagePolicyIds: string[]
+  packagePolicyIds: string[],
+  useBundledPackages = false
 ): Promise<UpgradeManagedPackagePoliciesResult[]> => {
   const results: UpgradeManagedPackagePoliciesResult[] = [];
 
@@ -46,6 +47,7 @@ export const upgradeManagedPackagePolicies = async (
       savedObjectsClient: soClient,
       pkgName: packagePolicy.package.name,
       pkgVersion: packagePolicy.package.version,
+      useBundledPackages,
     });
 
     const installedPackage = await getInstallation({

x-pack/plugins/fleet/server/services/package_policies_to_agent_permissions.ts

@@ -15,7 +15,8 @@ export const DEFAULT_CLUSTER_PERMISSIONS = ['monitor'];
 
 export async function storedPackagePoliciesToAgentPermissions(
   soClient: SavedObjectsClientContract,
-  packagePolicies: string[] | PackagePolicy[]
+  packagePolicies: string[] | PackagePolicy[],
+  options?: { useBundledPackages?: boolean }
 ): Promise<FullAgentPolicyOutputPermissions | undefined> {
   if (packagePolicies.length === 0) {
     return;
@@ -38,6 +39,7 @@ export async function storedPackagePoliciesToAgentPermissions(
         savedObjectsClient: soClient,
         pkgName: packagePolicy.package.name,
         pkgVersion: packagePolicy.package.version,
+        useBundledPackages: options?.useBundledPackages ?? false,
       });
 
       if (!pkg.data_streams || pkg.data_streams.length === 0) {

x-pack/plugins/fleet/server/services/package_policy.ts

@@ -43,21 +43,22 @@ import type {
   ListResult,
   UpgradePackagePolicyDryRunResponseItem,
   RegistryDataStream,
+  InstallablePackage,
 } from '../../common';
 import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants';
 import {
   IngestManagerError,
   ingestErrorToResponseOptions,
   PackagePolicyIneligibleForUpgradeError,
   PackagePolicyValidationError,
+  PackageCacheError,
 } from '../errors';
 import { NewPackagePolicySchema, UpdatePackagePolicySchema } from '../types';
 import type {
   NewPackagePolicy,
   UpdatePackagePolicy,
   PackagePolicy,
   PackagePolicySOAttributes,
-  RegistryPackage,
   DryRunPackagePolicy,
 } from '../types';
 import type { ExternalCallback } from '..';
@@ -73,6 +74,7 @@ import { appContextService } from '.';
 import { removeOldAssets } from './epm/packages/cleanup';
 import type { PackageUpdateEvent, UpdateEventType } from './upgrade_sender';
 import { sendTelemetryEvents } from './upgrade_sender';
+import { getArchivePackage } from './epm/archive';
 
 export type InputsOverride = Partial<NewPackagePolicyInput> & {
   vars?: Array<NewPackagePolicyInput['vars'] & { name: string }>;
@@ -103,6 +105,7 @@ class PackagePolicyService {
       skipEnsureInstalled?: boolean;
       skipUniqueNameVerification?: boolean;
       overwrite?: boolean;
+      useBundledPackages?: boolean;
     }
   ): Promise<PackagePolicy> {
     if (!options?.skipUniqueNameVerification) {
@@ -132,9 +135,11 @@ class PackagePolicyService {
         savedObjectsClient: soClient,
         pkgName: packagePolicy.package.name,
         pkgVersion: packagePolicy.package.version,
+        useBundledPackages: options?.useBundledPackages,
       });
 
-      let pkgInfo;
+      let pkgInfo: PackageInfo;
+
       if (options?.skipEnsureInstalled) pkgInfo = await pkgInfoPromise;
       else {
         const [, packageInfo] = await Promise.all([
@@ -162,16 +167,35 @@ class PackagePolicyService {
       }
       validatePackagePolicyOrThrow(packagePolicy, pkgInfo);
 
-      const registryPkgInfo = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version);
+      const installablePackage: InstallablePackage = await Registry.fetchInfo(
+        pkgInfo.name,
+        pkgInfo.version
+      ).catch(async (error) => {
+        if (!options?.useBundledPackages) {
+          throw error;
+        }
+
+        // If we're using bundled packages, try to pull the package info from ES instead
+        const archivePackage = await getArchivePackage({
+          name: pkgInfo.name,
+          version: pkgInfo.version,
+        });
+
+        if (!archivePackage) {
+          throw new PackageCacheError('Unable to find bundled package in cache');
+        }
+
+        return archivePackage?.packageInfo;
+      });
 
       inputs = await this._compilePackagePolicyInputs(
-        registryPkgInfo,
+        installablePackage,
         pkgInfo,
         packagePolicy.vars || {},
         inputs
       );
 
-      elasticsearch = registryPkgInfo.elasticsearch;
+      elasticsearch = installablePackage.elasticsearch;
     }
 
     const isoDate = new Date().toISOString();
@@ -798,14 +822,24 @@ class PackagePolicyService {
   }
 
   public async _compilePackagePolicyInputs(
-    registryPkgInfo: RegistryPackage,
+    installablePackage: InstallablePackage,
     pkgInfo: PackageInfo,
     vars: PackagePolicy['vars'],
     inputs: PackagePolicyInput[]
   ): Promise<PackagePolicyInput[]> {
     const inputsPromises = inputs.map(async (input) => {
-      const compiledInput = await _compilePackagePolicyInput(registryPkgInfo, pkgInfo, vars, input);
-      const compiledStreams = await _compilePackageStreams(registryPkgInfo, pkgInfo, vars, input);
+      const compiledInput = await _compilePackagePolicyInput(
+        installablePackage,
+        pkgInfo,
+        vars,
+        input
+      );
+      const compiledStreams = await _compilePackageStreams(
+        installablePackage,
+        pkgInfo,
+        vars,
+        input
+      );
       return {
         ...input,
         compiled_input: compiledInput,
@@ -916,7 +950,7 @@ function assignStreamIdToInput(packagePolicyId: string, input: NewPackagePolicyI
 }
 
 async function _compilePackagePolicyInput(
-  registryPkgInfo: RegistryPackage,
+  installablePackage: InstallablePackage,
   pkgInfo: PackageInfo,
   vars: PackagePolicy['vars'],
   input: PackagePolicyInput
@@ -941,7 +975,7 @@ async function _compilePackagePolicyInput(
     return undefined;
   }
 
-  const [pkgInputTemplate] = await getAssetsData(registryPkgInfo, (path: string) =>
+  const [pkgInputTemplate] = await getAssetsData(installablePackage, (path: string) =>
     path.endsWith(`/agent/input/${packageInput.template_path!}`)
   );
 
@@ -957,13 +991,13 @@ async function _compilePackagePolicyInput(
 }
 
 async function _compilePackageStreams(
-  registryPkgInfo: RegistryPackage,
+  installablePackage: InstallablePackage,
   pkgInfo: PackageInfo,
   vars: PackagePolicy['vars'],
   input: PackagePolicyInput
 ) {
   const streamsPromises = input.streams.map((stream) =>
-    _compilePackageStream(registryPkgInfo, pkgInfo, vars, input, stream)
+    _compilePackageStream(installablePackage, pkgInfo, vars, input, stream)
   );
 
   return await Promise.all(streamsPromises);
@@ -1006,7 +1040,7 @@ export function _applyIndexPrivileges(
 }
 
 async function _compilePackageStream(
-  registryPkgInfo: RegistryPackage,
+  installablePackage: InstallablePackage,
   pkgInfo: PackageInfo,
   vars: PackagePolicy['vars'],
   input: PackagePolicyInput,
@@ -1049,7 +1083,7 @@ async function _compilePackageStream(
   const datasetPath = packageDataStream.path;
 
   const [pkgStreamTemplate] = await getAssetsData(
-    registryPkgInfo,
+    installablePackage,
     (path: string) => path.endsWith(streamFromPkg.template_path),
     datasetPath
   );

x-pack/plugins/fleet/server/services/preconfiguration.ts

@@ -369,7 +369,8 @@ export async function ensurePreconfiguredPackagesAndPolicies(
   const packagePolicyUpgradeResults = await upgradeManagedPackagePolicies(
     soClient,
     esClient,
-    allPackagePolicyIds.items
+    allPackagePolicyIds.items,
+    true
   );
 
   return {
@@ -424,10 +425,14 @@ async function addPreconfiguredPolicyPackages(
 ) {
   // Add packages synchronously to avoid overwriting
   for (const { installedPackage, id, name, description, inputs } of installedPackagePolicies) {
+    // Allow requests to the registry to fall back to bundled packages if no registry is reachable in the calls below
+    const useBundledPackages = true;
+
     const packageInfo = await getPackageInfo({
       savedObjectsClient: soClient,
       pkgName: installedPackage.name,
       pkgVersion: installedPackage.version,
+      useBundledPackages,
     });
 
     await addPackageToAgentPolicy(
@@ -440,7 +445,8 @@ async function addPreconfiguredPolicyPackages(
       id,
       description,
       (policy) => preconfigurePackageInputs(policy, packageInfo, inputs),
-      bumpAgentPolicyRevison
+      bumpAgentPolicyRevison,
+      useBundledPackages
     );
   }
 }