elastic · kpollich · Feb 16, 2022 · Feb 9, 2022 · Feb 9, 2022 · Feb 9, 2022
@@ -107,7 +107,7 @@ export type InstallablePackage = RegistryPackage | ArchivePackage;
 
 export type ArchivePackage = PackageSpecManifest &
   // should an uploaded package be able to specify `internal`?
-  Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal'>;
+  Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal' | 'elasticsearch'>;
 
 export type RegistryPackage = PackageSpecManifest &
   Partial<RegistryOverridesToOptional> &

@@ -43,6 +43,7 @@ export class ConcurrentInstallOperationError extends IngestManagerError {}
 export class AgentReassignmentError extends IngestManagerError {}
 export class PackagePolicyIneligibleForUpgradeError extends IngestManagerError {}
 export class PackagePolicyValidationError extends IngestManagerError {}
+export class BundledPackageNotFoundError extends IngestManagerError {}
 export class HostedAgentPolicyRestrictionRelatedError extends IngestManagerError {
   constructor(message = 'Cannot perform that action') {
     super(

@@ -34,7 +34,7 @@ export async function getFullAgentPolicy(
   options?: { standalone: boolean }
 ): Promise<FullAgentPolicy | null> {
   let agentPolicy;
-  const standalone = options?.standalone;
+  const standalone = options?.standalone ?? false;
 
   try {
     agentPolicy = await agentPolicyService.get(soClient, id);

@@ -16,6 +16,7 @@ import type {
   RegistryElasticsearch,
   InstallablePackage,
   IndexTemplate,
+  PackageInfo,
 } from '../../../../types';
 import { loadFieldsFromYaml, processFields } from '../../fields/field';
 import type { Field } from '../../fields/field';
@@ -31,6 +32,8 @@ import type { ESAssetMetadata } from '../meta';
 import { getESAssetMetadata } from '../meta';
 import { retryTransientEsErrors } from '../retry';
 
+import { getPackageInfo } from '../../packages';
+
 import {
   generateMappings,
   generateTemplateName,
@@ -62,10 +65,16 @@ export const installTemplates = async (
   const dataStreams = installablePackage.data_streams;
   if (!dataStreams) return [];
 
+  const packageInfo = await getPackageInfo({
+    savedObjectsClient,
+    pkgName: installablePackage.name,
+    pkgVersion: installablePackage.version,
+  });
+
   const installedTemplatesNested = await Promise.all(
     dataStreams.map((dataStream) =>
       installTemplateForDataStream({
-        pkg: installablePackage,
+        pkg: packageInfo,
         esClient,
         logger,
         dataStream,
@@ -177,7 +186,7 @@ export async function installTemplateForDataStream({
   logger,
   dataStream,
 }: {
-  pkg: InstallablePackage;
+  pkg: PackageInfo;
   esClient: ElasticsearchClient;
   logger: Logger;
   dataStream: RegistryDataStream;

@@ -7,7 +7,7 @@
 
 import { safeLoad } from 'js-yaml';
 
-import type { InstallablePackage } from '../../../types';
+import type { PackageInfo } from '../../../types';
 import { getAssetsData } from '../packages/assets';
 
 // This should become a copy of https://github.com/elastic/beats/blob/d9a4c9c240a9820fab15002592e5bb6db318543b/libbeat/mapping/field.go#L39
@@ -261,7 +261,7 @@ const isFields = (path: string) => {
  */
 
 export const loadFieldsFromYaml = async (
-  pkg: InstallablePackage,
+  pkg: PackageInfo,
   datasetName?: string
 ): Promise<Field[]> => {
   // Fetch all field definition files

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { InstallablePackage } from '../../../types';
+import type { PackageInfo } from '../../../types';
 
 import { getArchiveFilelist } from '../archive/cache';
 
@@ -66,7 +66,7 @@ const tests = [
 test('testGetAssets', () => {
   for (const value of tests) {
     // as needed to pretend it is an InstallablePackage
-    const assets = getAssets(value.package as InstallablePackage, value.filter, value.dataset);
+    const assets = getAssets(value.package as PackageInfo, value.filter, value.dataset);
     expect(assets).toStrictEqual(value.expected);
   }
 });
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { InstallablePackage } from '../../../types';
+import type { PackageInfo } from '../../../types';
 import { getArchiveFilelist, getAsset } from '../archive';
 import type { ArchiveEntry } from '../archive';
 
@@ -17,7 +17,7 @@ import type { ArchiveEntry } from '../archive';
 // and different package and version structure
 
 export function getAssets(
-  packageInfo: InstallablePackage,
+  packageInfo: PackageInfo,
   filter = (path: string): boolean => true,
   datasetName?: string
 ): string[] {
@@ -52,7 +52,7 @@ export function getAssets(
 // ASK: Does getAssetsData need an installSource now?
 // if so, should it be an Installation vs InstallablePackage or add another argument?
 export async function getAssetsData(
-  packageInfo: InstallablePackage,
+  packageInfo: PackageInfo,
   filter = (path: string): boolean => true,
   datasetName?: string
 ): Promise<ArchiveEntry[]> {

@@ -14,7 +14,6 @@ import type { InstallResult } from '../../../types';
 
 import { installPackage, isPackageVersionOrLaterInstalled } from './install';
 import type { BulkInstallResponse, IBulkInstallPackageError } from './install';
-import { getBundledPackages } from './get_bundled_packages';
 
 interface BulkInstallPackagesParams {
   savedObjectsClient: SavedObjectsClientContract;
@@ -31,23 +30,23 @@ export async function bulkInstallPackages({
   esClient,
   spaceId,
   force,
-  preferredSource = 'registry',
 }: BulkInstallPackagesParams): Promise<BulkInstallResponse[]> {
   const logger = appContextService.getLogger();
 
-  const bundledPackages = await getBundledPackages();
-
   const packagesResults = await Promise.allSettled(
-    packagesToInstall.map((pkg) => {
-      if (typeof pkg === 'string') return Registry.fetchFindLatestPackage(pkg);
-      return Promise.resolve(pkg);
+    packagesToInstall.map(async (pkg) => {
+      if (typeof pkg !== 'string') {
+        return Promise.resolve(pkg);
+      }
+
+      return Registry.fetchFindLatestPackageWithFallbackToBundled(pkg);
     })
   );
 
   logger.debug(
-    `kicking off bulk install of ${packagesToInstall.join(
-      ', '
-    )} with preferred source of "${preferredSource}"`
+    `kicking off bulk install of ${packagesToInstall
+      .map((pkg) => (typeof pkg === 'string' ? pkg : pkg.name))
+      .join(', ')}`
   );
 
   const bulkInstallResults = await Promise.allSettled(
@@ -83,61 +82,16 @@ export async function bulkInstallPackages({
         };
       }
 
-      let installResult: InstallResult;
       const pkgkey = Registry.pkgToPkgKey(pkgKeyProps);
 
-      const bundledPackage = bundledPackages.find((pkg) => pkg.name === pkgkey);
-
-      // If preferred source is bundled packages on disk, attempt to install from disk first, then fall back to registry
-      if (preferredSource === 'bundled') {
-        if (bundledPackage) {
-          logger.debug(
-            `kicking off install of ${pkgKeyProps.name}-${pkgKeyProps.version} from bundled package on disk`
-          );
-          installResult = await installPackage({
-            savedObjectsClient,
-            esClient,
-            installSource: 'upload',
-            archiveBuffer: bundledPackage.buffer,
-            contentType: 'application/zip',
-            spaceId,
-          });
-        } else {
-          installResult = await installPackage({
-            savedObjectsClient,
-            esClient,
-            pkgkey,
-            installSource: 'registry',
-            spaceId,
-            force,
-          });
-        }
-      } else {
-        // If preferred source is registry, attempt to install from registry first, then fall back to bundled packages on disk
-        installResult = await installPackage({
-          savedObjectsClient,
-          esClient,
-          pkgkey,
-          installSource: 'registry',
-          spaceId,
-          force,
-        });
-
-        // If we initially errored, try to install from bundled package on disk
-        if (installResult.error && bundledPackage) {
-          logger.debug(
-            `kicking off install of ${pkgKeyProps.name}-${pkgKeyProps.version} from bundled package on disk`
-          );
-          installResult = await installPackage({
-            savedObjectsClient,
-            esClient,
-            installSource: 'upload',
-            archiveBuffer: bundledPackage.buffer,
-            contentType: 'application/zip',
-            spaceId,
-          });
-        }
-      }
+      const installResult = await installPackage({
+        savedObjectsClient,
+        esClient,
+        pkgkey,
+        installSource: 'registry',
+        spaceId,
+        force,
+      });
 
       if (installResult.error) {
         return {

@@ -143,9 +143,10 @@ export async function getPackageInfo(options: {
   pkgVersion: string;
 }): Promise<PackageInfo> {
   const { savedObjectsClient, pkgName, pkgVersion } = options;
+
   const [savedObject, latestPackage] = await Promise.all([
     getInstallationObject({ savedObjectsClient, pkgName }),
-    Registry.fetchFindLatestPackage(pkgName),
+    Registry.fetchFindLatestPackageWithFallbackToBundled(pkgName),
   ]);
 
   // If no package version is provided, use the installed version in the response

@@ -9,11 +9,13 @@ import path from 'path';
 import fs from 'fs/promises';
 
 import { appContextService } from '../../app_context';
+import { splitPkgKey } from '../registry';
 
 const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../bundled_packages');
 
 interface BundledPackage {
   name: string;
+  version: string;
   buffer: Buffer;
 }
 
@@ -26,8 +28,11 @@ export async function getBundledPackages(): Promise<BundledPackage[]> {
       zipFiles.map(async (zipFile) => {
         const file = await fs.readFile(path.join(BUNDLED_PACKAGE_DIRECTORY, zipFile));
 
+        const { pkgName, pkgVersion } = splitPkgKey(zipFile.replace(/\.zip$/, ''));
+
         return {
-          name: zipFile.replace(/\.zip$/, ''),
+          name: pkgName,
+          version: pkgVersion,
           buffer: file,
         };
       })

@@ -44,6 +44,7 @@ import { removeInstallation } from './remove';
 import { getPackageSavedObjects } from './get';
 import { _installPackage } from './_install_package';
 import { removeOldAssets } from './cleanup';
+import { getBundledPackages } from './get_bundled_packages';
 
 export async function isPackageInstalled(options: {
   savedObjectsClient: SavedObjectsClientContract;
@@ -468,8 +469,31 @@ export async function installPackage(args: InstallPackageParams) {
   const logger = appContextService.getLogger();
   const { savedObjectsClient, esClient } = args;
 
+  const bundledPackages = await getBundledPackages();
+
   if (args.installSource === 'registry') {
     const { pkgkey, force, spaceId } = args;
+
+    const matchingBundledPackage = bundledPackages.find(
+      (pkg) => Registry.pkgToPkgKey(pkg) === pkgkey
+    );
+
+    if (matchingBundledPackage) {
+      logger.debug(
+        `found bundled package for requested install of ${pkgkey} - installing from bundled package archive`
+      );
+
+      const response = installPackageByUpload({
+        savedObjectsClient,
+        esClient,
+        archiveBuffer: matchingBundledPackage.buffer,
+        contentType: 'application/zip',
+        spaceId,
+      });
+
+      return response;
+    }
+
     logger.debug(`kicking off install of ${pkgkey} from registry`);
     const response = installPackageFromRegistry({
       savedObjectsClient,

@@ -35,6 +35,8 @@ import { streamToBuffer } from '../streams';
 import { appContextService } from '../..';
 import { PackageNotFoundError, PackageCacheError, RegistryResponseError } from '../../../errors';
 
+import { getBundledPackages } from '../packages/get_bundled_packages';
+
 import { fetchUrl, getResponse, getResponseStream } from './requests';
 import { getRegistryUrl } from './registry_url';
 
@@ -71,7 +73,7 @@ export async function fetchFindLatestPackage(packageName: string): Promise<Regis
 
   setKibanaVersion(url);
 
-  const res = await fetchUrl(url.toString());
+  const res = await fetchUrl(url.toString(), 1);
   const searchResults = JSON.parse(res);
   if (searchResults.length) {
     return searchResults[0];
@@ -80,6 +82,23 @@ export async function fetchFindLatestPackage(packageName: string): Promise<Regis
   }
 }
 
+export async function fetchFindLatestPackageWithFallbackToBundled(pkgName: string) {
+  return fetchFindLatestPackage(pkgName).catch(async (error) => {
+    const bundledPackages = await getBundledPackages();
+    const bundledPackage = bundledPackages.find((b) => b.name === pkgName);
+
+    // If we don't find a bundled package, re-throw the original error
+    if (!bundledPackage) {
+      throw error;
+    }
+
+    return {
+      name: bundledPackage.name,
+      version: bundledPackage.version,
+    };
+  });
+}
+
 export async function fetchInfo(pkgName: string, pkgVersion: string): Promise<RegistryPackage> {
   const registryUrl = getRegistryUrl();
   try {