elastic · kpollich · Feb 16, 2022 · Feb 9, 2022 · Feb 9, 2022 · Feb 9, 2022
@@ -107,7 +107,7 @@ export type InstallablePackage = RegistryPackage | ArchivePackage;
 
 export type ArchivePackage = PackageSpecManifest &
   // should an uploaded package be able to specify `internal`?
-  Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal'>;
+  Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal' | 'elasticsearch'>;
 
 export type RegistryPackage = PackageSpecManifest &
   Partial<RegistryOverridesToOptional> &

@@ -43,6 +43,7 @@ export class ConcurrentInstallOperationError extends IngestManagerError {}
 export class AgentReassignmentError extends IngestManagerError {}
 export class PackagePolicyIneligibleForUpgradeError extends IngestManagerError {}
 export class PackagePolicyValidationError extends IngestManagerError {}
+export class BundledPackageNotFoundError extends IngestManagerError {}
 export class HostedAgentPolicyRestrictionRelatedError extends IngestManagerError {
   constructor(message = 'Cannot perform that action') {
     super(

@@ -34,7 +34,7 @@ export async function getFullAgentPolicy(
   options?: { standalone: boolean }
 ): Promise<FullAgentPolicy | null> {
   let agentPolicy;
-  const standalone = options?.standalone;
+  const standalone = options?.standalone ?? false;
 
   try {
     agentPolicy = await agentPolicyService.get(soClient, id);

@@ -12,6 +12,8 @@ import * as Registry from '../registry';
 
 import type { InstallResult } from '../../../types';
 
+import { BundledPackageNotFoundError } from '../../../errors';
+
 import { installPackage, isPackageVersionOrLaterInstalled } from './install';
 import type { BulkInstallResponse, IBulkInstallPackageError } from './install';
 import { getBundledPackages } from './get_bundled_packages';
@@ -39,8 +41,32 @@ export async function bulkInstallPackages({
 
   const packagesResults = await Promise.allSettled(
     packagesToInstall.map((pkg) => {
-      if (typeof pkg === 'string') return Registry.fetchFindLatestPackage(pkg);
-      return Promise.resolve(pkg);
+      if (typeof pkg !== 'string') {
+        return Promise.resolve(pkg);
+      }
+
+      return Registry.fetchFindLatestPackage(pkg).catch((error) => {
+        // For registry preferred bulk installs, consider an inability to reach the configured
+        // registry a fotal error
+        if (preferredSource === 'registry') {
+          return Promise.reject(error);
+        }
+
+        // For bundled preferred bulk installs, we want to allow the installation to continue, even
+        // if the registry is not reachable
+        const bundledPackage = bundledPackages.find((b) => b.name === pkg);
+
+        if (!bundledPackage) {
+          return Promise.reject(
+            new BundledPackageNotFoundError(`No bundled package found with name ${pkg}`)
+          );
+        }
+
+        return Promise.resolve({
+          name: bundledPackage.name,
+          version: bundledPackage.version,
+        });
+      });
     })
   );
 
@@ -86,7 +112,7 @@ export async function bulkInstallPackages({
       let installResult: InstallResult;
       const pkgkey = Registry.pkgToPkgKey(pkgKeyProps);
 
-      const bundledPackage = bundledPackages.find((pkg) => pkg.name === pkgkey);
+      const bundledPackage = bundledPackages.find((pkg) => `${pkg.name}-${pkg.version}` === pkgkey);
 
       // If preferred source is bundled packages on disk, attempt to install from disk first, then fall back to registry
       if (preferredSource === 'bundled') {

@@ -21,14 +21,15 @@ import type {
   GetCategoriesRequest,
 } from '../../../../common/types';
 import type { Installation, PackageInfo } from '../../../types';
-import { IngestManagerError } from '../../../errors';
+import { BundledPackageNotFoundError, IngestManagerError } from '../../../errors';
 import { appContextService } from '../../';
 import * as Registry from '../registry';
 import { getEsPackage } from '../archive/storage';
 import { getArchivePackage } from '../archive';
 import { normalizeKuery } from '../../saved_object';
 
 import { createInstallableFrom } from './index';
+import { getBundledPackages } from './get_bundled_packages';
 
 export type { SearchParams } from '../registry';
 export { getFile } from '../registry';
@@ -106,7 +107,24 @@ export async function getPackageInfo(options: {
   const { savedObjectsClient, pkgName, pkgVersion } = options;
   const [savedObject, latestPackage] = await Promise.all([
     getInstallationObject({ savedObjectsClient, pkgName }),
-    Registry.fetchFindLatestPackage(pkgName),
+    new Promise<{ name: string; version: string }>((resolve, reject) => {
+      return Registry.fetchFindLatestPackage(pkgName).catch(async (error) => {
+        const bundledPackages = await getBundledPackages();
+        const bundledPackage = bundledPackages.find(
+          (b) => b.name === pkgName && b.version === pkgVersion
+        );
+
+        // If we don't find a bundled package, reject with the original error
+        if (!bundledPackage) {
+          return reject(error);
+        }
+
+        resolve({
+          name: bundledPackage.name,
+          version: bundledPackage.version,
+        });
+      });
+    }),
   ]);
 
   // If no package version is provided, use the installed version in the response

@@ -9,11 +9,13 @@ import path from 'path';
 import fs from 'fs/promises';
 
 import { appContextService } from '../../app_context';
+import { splitPkgKey } from '../registry';
 
 const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../bundled_packages');
 
 interface BundledPackage {
   name: string;
+  version: string;
   buffer: Buffer;
 }
 
@@ -26,8 +28,11 @@ export async function getBundledPackages(): Promise<BundledPackage[]> {
       zipFiles.map(async (zipFile) => {
         const file = await fs.readFile(path.join(BUNDLED_PACKAGE_DIRECTORY, zipFile));
 
+        const { pkgName, pkgVersion } = splitPkgKey(zipFile.replace(/\.zip$/, ''));
+
         return {
-          name: zipFile.replace(/\.zip$/, ''),
+          name: pkgName,
+          version: pkgVersion,
           buffer: file,
         };
       })

@@ -71,7 +71,7 @@ export async function fetchFindLatestPackage(packageName: string): Promise<Regis
 
   setKibanaVersion(url);
 
-  const res = await fetchUrl(url.toString());
+  const res = await fetchUrl(url.toString(), 1);
   const searchResults = JSON.parse(res);
   if (searchResults.length) {
     return searchResults[0];

@@ -34,13 +34,13 @@ async function registryFetch(url: string) {
   }
 }
 
-export async function getResponse(url: string): Promise<Response> {
+export async function getResponse(url: string, retries: number = 5): Promise<Response> {
   try {
     // we only want to retry certain failures like network issues
     // the rest should only try the one time then fail as they do now
     const response = await pRetry(() => registryFetch(url), {
       factor: 2,
-      retries: 5,
+      retries,
       onFailedAttempt: (error) => {
         // we only want to retry certain types of errors, like `ECONNREFUSED` and other operational errors
         // and let the others through without retrying
@@ -67,13 +67,16 @@ export async function getResponse(url: string): Promise<Response> {
   }
 }
 
-export async function getResponseStream(url: string): Promise<NodeJS.ReadableStream> {
-  const res = await getResponse(url);
+export async function getResponseStream(
+  url: string,
+  retries?: number
+): Promise<NodeJS.ReadableStream> {
+  const res = await getResponse(url, retries);
   return res.body;
 }
 
-export async function fetchUrl(url: string): Promise<string> {
-  return getResponseStream(url).then(streamToString);
+export async function fetchUrl(url: string, retries?: number): Promise<string> {
+  return getResponseStream(url, retries).then(streamToString);
 }
 
 // node-fetch throws a FetchError for those types of errors and

@@ -43,6 +43,7 @@ import type {
   ListResult,
   UpgradePackagePolicyDryRunResponseItem,
   RegistryDataStream,
+  InstallablePackage,
 } from '../../common';
 import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants';
 import {
@@ -57,7 +58,6 @@ import type {
   UpdatePackagePolicy,
   PackagePolicy,
   PackagePolicySOAttributes,
-  RegistryPackage,
   DryRunPackagePolicy,
 } from '../types';
 import type { ExternalCallback } from '..';
@@ -73,6 +73,7 @@ import { appContextService } from '.';
 import { removeOldAssets } from './epm/packages/cleanup';
 import type { PackageUpdateEvent, UpdateEventType } from './upgrade_sender';
 import { sendTelemetryEvents } from './upgrade_sender';
+import { getArchivePackage } from './epm/archive';
 
 export type InputsOverride = Partial<NewPackagePolicyInput> & {
   vars?: Array<NewPackagePolicyInput['vars'] & { name: string }>;
@@ -134,7 +135,8 @@ class PackagePolicyService {
         pkgVersion: packagePolicy.package.version,
       });
 
-      let pkgInfo;
+      let pkgInfo: PackageInfo;
+
       if (options?.skipEnsureInstalled) pkgInfo = await pkgInfoPromise;
       else {
         const [, packageInfo] = await Promise.all([
@@ -162,16 +164,33 @@ class PackagePolicyService {
       }
       validatePackagePolicyOrThrow(packagePolicy, pkgInfo);
 
-      const registryPkgInfo = await Registry.fetchInfo(pkgInfo.name, pkgInfo.version);
+      const installablePackage: InstallablePackage = await Registry.fetchInfo(
+        pkgInfo.name,
+        pkgInfo.version
+      ).catch(async (error) => {
+        // Attempt to get an ArchivePackage object from the package info cache if the registry
+        // is not reachable. This will pull a bundled package if it was previously installed.
+        const archivePackage = await getArchivePackage({
+          name: pkgInfo.name,
+          version: pkgInfo.version,
+        });
+
+        // If we don't find a package in the cache, throw the original error
+        if (!archivePackage) {
+          throw error;
+        }
+
+        return archivePackage?.packageInfo;
+      });
 
       inputs = await this._compilePackagePolicyInputs(
-        registryPkgInfo,
+        installablePackage,
         pkgInfo,
         packagePolicy.vars || {},
         inputs
       );
 
-      elasticsearch = registryPkgInfo.elasticsearch;
+      elasticsearch = installablePackage.elasticsearch;
     }
 
     const isoDate = new Date().toISOString();
@@ -798,14 +817,24 @@ class PackagePolicyService {
   }
 
   public async _compilePackagePolicyInputs(
-    registryPkgInfo: RegistryPackage,
+    installablePackage: InstallablePackage,
     pkgInfo: PackageInfo,
     vars: PackagePolicy['vars'],
     inputs: PackagePolicyInput[]
   ): Promise<PackagePolicyInput[]> {
     const inputsPromises = inputs.map(async (input) => {
-      const compiledInput = await _compilePackagePolicyInput(registryPkgInfo, pkgInfo, vars, input);
-      const compiledStreams = await _compilePackageStreams(registryPkgInfo, pkgInfo, vars, input);
+      const compiledInput = await _compilePackagePolicyInput(
+        installablePackage,
+        pkgInfo,
+        vars,
+        input
+      );
+      const compiledStreams = await _compilePackageStreams(
+        installablePackage,
+        pkgInfo,
+        vars,
+        input
+      );
       return {
         ...input,
         compiled_input: compiledInput,
@@ -916,7 +945,7 @@ function assignStreamIdToInput(packagePolicyId: string, input: NewPackagePolicyI
 }
 
 async function _compilePackagePolicyInput(
-  registryPkgInfo: RegistryPackage,
+  installablePackage: InstallablePackage,
   pkgInfo: PackageInfo,
   vars: PackagePolicy['vars'],
   input: PackagePolicyInput
@@ -941,7 +970,7 @@ async function _compilePackagePolicyInput(
     return undefined;
   }
 
-  const [pkgInputTemplate] = await getAssetsData(registryPkgInfo, (path: string) =>
+  const [pkgInputTemplate] = await getAssetsData(installablePackage, (path: string) =>
     path.endsWith(`/agent/input/${packageInput.template_path!}`)
   );
 
@@ -957,13 +986,13 @@ async function _compilePackagePolicyInput(
 }
 
 async function _compilePackageStreams(
-  registryPkgInfo: RegistryPackage,
+  installablePackage: InstallablePackage,
   pkgInfo: PackageInfo,
   vars: PackagePolicy['vars'],
   input: PackagePolicyInput
 ) {
   const streamsPromises = input.streams.map((stream) =>
-    _compilePackageStream(registryPkgInfo, pkgInfo, vars, input, stream)
+    _compilePackageStream(installablePackage, pkgInfo, vars, input, stream)
   );
 
   return await Promise.all(streamsPromises);
@@ -1006,7 +1035,7 @@ export function _applyIndexPrivileges(
 }
 
 async function _compilePackageStream(
-  registryPkgInfo: RegistryPackage,
+  installablePackage: InstallablePackage,
   pkgInfo: PackageInfo,
   vars: PackagePolicy['vars'],
   input: PackagePolicyInput,
@@ -1049,7 +1078,7 @@ async function _compilePackageStream(
   const datasetPath = packageDataStream.path;
 
   const [pkgStreamTemplate] = await getAssetsData(
-    registryPkgInfo,
+    installablePackage,
     (path: string) => path.endsWith(streamFromPkg.template_path),
     datasetPath
   );

@@ -164,12 +164,6 @@ jest.mock('./epm/packages/install', () => ({
       // Treat the buffer value passed in tests as the package's name for simplicity
       const pkgName = archiveBuffer.toString('utf8');
 
-      const installedPackage = mockInstalledPackages.get(pkgName);
-
-      if (installedPackage) {
-        return installedPackage;
-      }
-
       // Just install every bundled package at version '1.0.0'
       const packageInstallation = { name: pkgName, version: '1.0.0', title: pkgName };
       mockInstalledPackages.set(pkgName, packageInstallation);
@@ -743,11 +737,13 @@ describe('policy preconfiguration', () => {
       mockedGetBundledPackages.mockResolvedValue([
         {
           name: 'test_package',
+          version: '1.0.0',
           buffer: Buffer.from('test_package'),
         },
 
         {
           name: 'test_package_2',
+          version: '1.0.0',
           buffer: Buffer.from('test_package_2'),
         },
       ]);
@@ -784,6 +780,7 @@ describe('policy preconfiguration', () => {
           mockedGetBundledPackages.mockResolvedValue([
             {
               name: 'test_package',
+              version: '1.0.0',
               buffer: Buffer.from('test_package'),
             },
           ]);
@@ -823,6 +820,7 @@ describe('policy preconfiguration', () => {
           mockedGetBundledPackages.mockResolvedValue([
             {
               name: 'test_package',
+              version: '1.0.0',
               buffer: Buffer.from('test_package'),
             },
           ]);