[Fleet] Avoid breaking setup when compatible package is not available in registry

x-pack/plugins/fleet/common/openapi/bundled.json

@@ -646,6 +646,9 @@
                 "properties": {
                   "force": {
                     "type": "boolean"
+                  },
+                  "ignore_constraints": {
+                    "type": "boolean"
                   }
                 }
               }

x-pack/plugins/fleet/common/openapi/bundled.yaml

@@ -396,6 +396,8 @@ paths:
               properties:
                 force:
                   type: boolean
+                ignore_constraints:
+                  type: boolean
     put:
       summary: Packages - Update
       tags: []

x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml

@@ -78,6 +78,8 @@ post:
           properties:
             force:
               type: boolean
+            ignore_constraints:
+              type: boolean
 put:
   summary: Packages - Update
   tags: []

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -265,6 +265,7 @@ export const installPackageFromRegistryHandler: FleetRequestHandler<
     esClient,
     spaceId,
     force: request.body?.force,
+    ignoreConstraints: request.body?.ignore_constraints,
   });
   if (!res.error) {
     const body: InstallPackageResponse = {

x-pack/plugins/fleet/server/services/epm/packages/get.test.ts

@@ -19,6 +19,8 @@ import * as Registry from '../registry';
 import { createAppContextStartContractMock } from '../../../mocks';
 import { appContextService } from '../../app_context';
 
+import { PackageNotFoundError } from '../../../errors';
+
 import { getPackageInfo, getPackageUsageStats } from './get';
 
 const MockRegistry = Registry as jest.Mocked<typeof Registry>;
@@ -279,5 +281,45 @@ describe('When using EPM `get` services', () => {
         });
       });
     });
+
+    describe('registry fetch errors', () => {
+      it('throws when a package that is not installed is not available in the registry', async () => {
+        MockRegistry.fetchFindLatestPackage.mockResolvedValue(undefined);
+        const soClient = savedObjectsClientMock.create();
+        soClient.get.mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError());
+
+        await expect(
+          getPackageInfo({
+            savedObjectsClient: soClient,
+            pkgName: 'my-package',
+            pkgVersion: '1.0.0',
+          })
+        ).rejects.toThrowError(PackageNotFoundError);
+      });
+
+      it('sets the latestVersion to installed version when an installed package is not available in the registry', async () => {
+        MockRegistry.fetchFindLatestPackage.mockResolvedValue(undefined);
+        const soClient = savedObjectsClientMock.create();
+        soClient.get.mockResolvedValue({
+          id: 'my-package',
+          type: PACKAGES_SAVED_OBJECT_TYPE,
+          references: [],
+          attributes: {
+            install_status: 'installed',
+          },
+        });
+
+        await expect(
+          getPackageInfo({
+            savedObjectsClient: soClient,
+            pkgName: 'my-package',
+            pkgVersion: '1.0.0',
+          })
+        ).resolves.toMatchObject({
+          latestVersion: '1.0.0',
+          status: 'installed',
+        });
+      });
+    });
   });
 });

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -21,7 +21,7 @@ import type {
   GetCategoriesRequest,
 } from '../../../../common/types';
 import type { Installation, PackageInfo } from '../../../types';
-import { IngestManagerError } from '../../../errors';
+import { IngestManagerError, PackageNotFoundError } from '../../../errors';
 import { appContextService } from '../../';
 import * as Registry from '../registry';
 import { getEsPackage } from '../archive/storage';
@@ -145,17 +145,17 @@ export async function getPackageInfo(options: {
   const { savedObjectsClient, pkgName, pkgVersion } = options;
   const [savedObject, latestPackage] = await Promise.all([
     getInstallationObject({ savedObjectsClient, pkgName }),
-    Registry.fetchFindLatestPackage(pkgName),
+    Registry.fetchFindLatestPackage(pkgName, { throwIfNotFound: false }),
   ]);
 
-  // If no package version is provided, use the installed version in the response
-  let responsePkgVersion = pkgVersion || savedObject?.attributes.install_version;
-
-  // If no installed version of the given package exists, default to the latest version of the package
-  if (!responsePkgVersion) {
-    responsePkgVersion = latestPackage.version;
+  if (!savedObject && !latestPackage) {
+    throw new PackageNotFoundError(`[${pkgName}] package not installed or found in registry`);
   }
 
+  // If no package version is provided, use the installed version in the response, fallback to package from registry
+  const responsePkgVersion =
+    pkgVersion ?? savedObject?.attributes.install_version ?? latestPackage!.version;
+
   const getPackageRes = await getPackageFromSource({
     pkgName,
     pkgVersion: responsePkgVersion,
@@ -166,7 +166,7 @@ export async function getPackageInfo(options: {
 
   // add properties that aren't (or aren't yet) on the package
   const additions: EpmPackageAdditions = {
-    latestVersion: latestPackage.version,
+    latestVersion: latestPackage?.version ?? responsePkgVersion,
     title: packageInfo.title || nameAsTitle(packageInfo.name),
     assets: Registry.groupPathsByService(paths || []),
     removable: true,

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -205,6 +205,7 @@ interface InstallRegistryPackageParams {
   esClient: ElasticsearchClient;
   spaceId: string;
   force?: boolean;
+  ignoreConstraints?: boolean;
 }
 
 function getTelemetryEvent(pkgName: string, pkgVersion: string): PackageUpdateEvent {
@@ -233,6 +234,7 @@ async function installPackageFromRegistry({
   esClient,
   spaceId,
   force = false,
+  ignoreConstraints = false,
 }: InstallRegistryPackageParams): Promise<InstallResult> {
   const logger = appContextService.getLogger();
   // TODO: change epm API to /packageName/version so we don't need to do this
@@ -249,7 +251,7 @@ async function installPackageFromRegistry({
     installType = getInstallType({ pkgVersion, installedPkg });
 
     // get latest package version
-    const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
+    const latestPackage = await Registry.fetchFindLatestPackage(pkgName, { ignoreConstraints });
 
     // let the user install if using the force flag or needing to reinstall or install a previous version due to failed update
     const installOutOfDateVersionOk =
@@ -469,14 +471,15 @@ export async function installPackage(args: InstallPackageParams) {
   const { savedObjectsClient, esClient } = args;
 
   if (args.installSource === 'registry') {
-    const { pkgkey, force, spaceId } = args;
+    const { pkgkey, force, ignoreConstraints, spaceId } = args;
     logger.debug(`kicking off install of ${pkgkey} from registry`);
     const response = installPackageFromRegistry({
       savedObjectsClient,
       pkgkey,
       esClient,
       spaceId,
       force,
+      ignoreConstraints,
     });
     return response;
   } else if (args.installSource === 'upload') {

x-pack/plugins/fleet/server/services/epm/registry/index.ts

@@ -65,18 +65,33 @@ export async function fetchList(params?: SearchParams): Promise<RegistrySearchRe
   return fetchUrl(url.toString()).then(JSON.parse);
 }
 
-export async function fetchFindLatestPackage(packageName: string): Promise<RegistrySearchResult> {
+// When `throwIfNotFound` is true or undefined, return type will never be undefined.
+export async function fetchFindLatestPackage(
+  packageName: string,
+  options?: { ignoreConstraints?: boolean; throwIfNotFound?: true }
+): Promise<RegistrySearchResult>;
+export async function fetchFindLatestPackage(
+  packageName: string,
+  options: { ignoreConstraints?: boolean; throwIfNotFound: false }
+): Promise<RegistrySearchResult | undefined>;
+export async function fetchFindLatestPackage(
+  packageName: string,
+  options?: { ignoreConstraints?: boolean; throwIfNotFound?: boolean }
+): Promise<RegistrySearchResult | undefined> {
+  const { ignoreConstraints = false, throwIfNotFound = true } = options ?? {};
   const registryUrl = getRegistryUrl();
   const url = new URL(`${registryUrl}/search?package=${packageName}&experimental=true`);
 
-  setKibanaVersion(url);
+  if (!ignoreConstraints) {
+    setKibanaVersion(url);
+  }
 
   const res = await fetchUrl(url.toString());
   const searchResults = JSON.parse(res);
   if (searchResults.length) {
     return searchResults[0];
-  } else {
-    throw new PackageNotFoundError(`${packageName} not found`);
+  } else if (throwIfNotFound) {
+    throw new PackageNotFoundError(`[${packageName}] package not found in registry`);
   }
 }
 

x-pack/plugins/fleet/server/types/rest_spec/epm.ts

@@ -74,7 +74,8 @@ export const InstallPackageFromRegistryRequestSchema = {
   }),
   body: schema.nullable(
     schema.object({
-      force: schema.boolean(),
+      force: schema.boolean({ defaultValue: false }),
+      ignore_constraints: schema.boolean({ defaultValue: false }),
     })
   ),
 };

x-pack/test/fleet_api_integration/apis/epm/setup.ts

@@ -52,6 +52,40 @@ export default function (providerContext: FtrProviderContext) {
       });
     });
 
+    it('does not fail when package is no longer compatible in registry', async () => {
+      await supertest
+        .post(`/api/fleet/epm/packages/deprecated/0.1.0`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({ force: true, ignore_constraints: true })
+        .expect(200);
+
+      const agentPolicyResponse = await supertest
+        .post(`/api/fleet/agent_policies`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          name: 'deprecated-ap-1',
+          namespace: 'default',
+          monitoring_enabled: [],
+        })
+        .expect(200);
+
+      await supertest
+        .post(`/api/fleet/package_policies`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          name: 'deprecated-1',
+          policy_id: agentPolicyResponse.body.item.id,
+          package: {
+            name: 'deprecated',
+            version: '0.1.0',
+          },
+          inputs: [],
+        })
+        .expect(200);
+
+      await supertest.post('/api/fleet/setup').set('kbn-xsrf', 'xxxx').expect(200);
+    });
+
     it('allows elastic/fleet-server user to call required APIs', async () => {
       const {
         token,

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/deprecated/0.1.0/data_stream/test/fields/fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/deprecated/0.1.0/data_stream/test/manifest.yml

@@ -0,0 +1,9 @@
+title: Test Dataset
+
+type: logs
+
+elasticsearch:
+  index_template.mappings:
+    dynamic: false
+  index_template.settings:
+    index.lifecycle.name: reference

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/deprecated/0.1.0/docs/README.md

@@ -0,0 +1,3 @@
+# Test package
+
+This is a test package for testing installing or updating to an out-of-date package

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/deprecated/0.1.0/manifest.yml

@@ -0,0 +1,16 @@
+format_version: 1.0.0
+name: deprecated
+title: Package install/update test
+description: This is a package for testing deprecated packages
+version: 0.1.0
+categories: []
+release: beta
+type: integration
+license: basic
+
+conditions:
+  # Version number is not compatible with current version
+  elasticsearch:
+    version: '^1.0.0'
+  kibana:
+    version: '^1.0.0'