EMT-179: implement metadata query versioning based on ingest manager …

…installed ES assets (#77252)

* EMT-179: initial refactor for versioning

* EMT-179: move things before pulling from master

* EMT-179: fix build

* EMT-179: clean up

* EMT-179: add ingest hook, and improve all tests

* EMT-179: fix build

* EMT-179: clean up

* EMT-179: fix build

* EMT-179: fix build

* EMT-179: clean up

* EMT-179: more clean up

* EMT-179: clean up

* EMT-179: fix build

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/ingest_manager/server/index.ts

@@ -11,7 +11,7 @@ import {
   AGENT_POLICY_ROLLOUT_RATE_LIMIT_REQUEST_PER_INTERVAL,
   AGENT_POLLING_REQUEST_TIMEOUT_MS,
 } from '../common';
-export { AgentService, ESIndexPatternService, getRegistryUrl } from './services';
+export { AgentService, ESIndexPatternService, getRegistryUrl, PackageService } from './services';
 export {
   IngestManagerSetupContract,
   IngestManagerSetupDeps,

x-pack/plugins/ingest_manager/server/plugin.ts

@@ -13,6 +13,7 @@ import {
   PluginInitializerContext,
   SavedObjectsServiceStart,
   HttpServiceSetup,
+  SavedObjectsClientContract,
 } from 'kibana/server';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import { LicensingPluginSetup, ILicense } from '../../licensing/server';
@@ -47,14 +48,15 @@ import {
   registerSettingsRoutes,
   registerAppRoutes,
 } from './routes';
-import { IngestManagerConfigType, NewPackagePolicy } from '../common';
+import { EsAssetReference, IngestManagerConfigType, NewPackagePolicy } from '../common';
 import {
   appContextService,
   licenseService,
   ESIndexPatternSavedObjectService,
   ESIndexPatternService,
   AgentService,
   packagePolicyService,
+  PackageService,
 } from './services';
 import {
   getAgentStatusById,
@@ -65,6 +67,7 @@ import {
 import { CloudSetup } from '../../cloud/server';
 import { agentCheckinState } from './services/agents/checkin/state';
 import { registerIngestManagerUsageCollector } from './collectors/register';
+import { getInstallation } from './services/epm/packages';
 
 export interface IngestManagerSetupDeps {
   licensing: LicensingPluginSetup;
@@ -118,6 +121,7 @@ export type ExternalCallbacksStorage = Map<ExternalCallback[0], Set<ExternalCall
  */
 export interface IngestManagerStartContract {
   esIndexPatternService: ESIndexPatternService;
+  packageService: PackageService;
   agentService: AgentService;
   /**
    * Services for Ingest's package policies
@@ -273,6 +277,15 @@ export class IngestManagerPlugin
 
     return {
       esIndexPatternService: new ESIndexPatternSavedObjectService(),
+      packageService: {
+        getInstalledEsAssetReferences: async (
+          savedObjectsClient: SavedObjectsClientContract,
+          pkgName: string
+        ): Promise<EsAssetReference[]> => {
+          const installation = await getInstallation({ savedObjectsClient, pkgName });
+          return installation?.installed_es || [];
+        },
+      },
       agentService: {
         getAgent,
         listAgents,

x-pack/plugins/ingest_manager/server/services/index.ts

@@ -5,7 +5,7 @@
  */
 
 import { SavedObjectsClientContract, KibanaRequest } from 'kibana/server';
-import { AgentStatus, Agent } from '../types';
+import { AgentStatus, Agent, EsAssetReference } from '../types';
 import * as settingsService from './settings';
 export { ESIndexPatternSavedObjectService } from './es_index_pattern';
 
@@ -22,6 +22,17 @@ export interface ESIndexPatternService {
   ): Promise<string | undefined>;
 }
 
+/**
+ * Service that provides exported function that return information about EPM packages
+ */
+
+export interface PackageService {
+  getInstalledEsAssetReferences(
+    savedObjectsClient: SavedObjectsClientContract,
+    pkgName: string
+  ): Promise<EsAssetReference[]>;
+}
+
 /**
  * A service that provides exported functions that return information about an Agent
  */

x-pack/plugins/security_solution/common/endpoint/constants.ts

@@ -8,6 +8,7 @@ export const eventsIndexPattern = 'logs-endpoint.events.*';
 export const alertsIndexPattern = 'logs-endpoint.alerts-*';
 export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
 export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current-*';
+export const metadataTransformPrefix = 'metrics-endpoint.metadata-current-default';
 export const policyIndexPattern = 'metrics-endpoint.policy-*';
 export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
 export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency';

x-pack/plugins/security_solution/common/endpoint/types/index.ts

@@ -297,6 +297,8 @@ export interface HostResultList {
   request_page_size: number;
   /* the page index requested */
   request_page_index: number;
+  /* the version of the query strategy */
+  query_strategy_version: MetadataQueryStrategyVersions;
 }
 
 /**
@@ -504,9 +506,16 @@ export enum HostStatus {
   UNENROLLING = 'unenrolling',
 }
 
+export enum MetadataQueryStrategyVersions {
+  VERSION_1 = 'v1',
+  VERSION_2 = 'v2',
+}
+
 export type HostInfo = Immutable<{
   metadata: HostMetadata;
   host_status: HostStatus;
+  /* the version of the query strategy */
+  query_strategy_version: MetadataQueryStrategyVersions;
 }>;
 
 export type HostMetadataDetails = Immutable<{

x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/mock_endpoint_result_list.ts

@@ -11,6 +11,7 @@ import {
   HostPolicyResponse,
   HostResultList,
   HostStatus,
+  MetadataQueryStrategyVersions,
 } from '../../../../../common/endpoint/types';
 import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data';
 import {
@@ -49,13 +50,15 @@ export const mockEndpointResultList: (options?: {
     hosts.push({
       metadata: generator.generateHostMetadata(),
       host_status: HostStatus.ERROR,
+      query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
     });
   }
   const mock: HostResultList = {
     hosts,
     total,
     request_page_size: requestPageSize,
     request_page_index: requestPageIndex,
+    query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
   };
   return mock;
 };
@@ -67,6 +70,7 @@ export const mockEndpointDetailsApiResult = (): HostInfo => {
   return {
     metadata: generator.generateHostMetadata(),
     host_status: HostStatus.ERROR,
+    query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
   };
 };
 
@@ -103,6 +107,7 @@ const endpointListApiPathHandlerMocks = ({
         request_page_size: 10,
         request_page_index: 0,
         total: endpointsResults?.length || 0,
+        query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
       };
     },
 

x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.test.tsx

@@ -20,6 +20,7 @@ import {
   HostPolicyResponseActionStatus,
   HostPolicyResponseAppliedAction,
   HostStatus,
+  MetadataQueryStrategyVersions,
 } from '../../../../../common/endpoint/types';
 import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data';
 import { POLICY_STATUS_TO_HEALTH_COLOR, POLICY_STATUS_TO_TEXT } from './host_constants';
@@ -131,6 +132,7 @@ describe('when on the list page', () => {
               hostListData[index] = {
                 metadata: hostListData[index].metadata,
                 host_status: status,
+                query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
               };
             }
           );
@@ -301,6 +303,8 @@ describe('when on the list page', () => {
         // eslint-disable-next-line @typescript-eslint/naming-convention
         host_status,
         metadata: { host, ...details },
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        query_strategy_version,
       } = mockEndpointDetailsApiResult();
 
       hostDetails = {
@@ -312,6 +316,7 @@ describe('when on the list page', () => {
             id: '1',
           },
         },
+        query_strategy_version,
       };
 
       agentId = hostDetails.metadata.elastic.agent.id;
@@ -681,6 +686,7 @@ describe('when on the list page', () => {
       hostInfo = {
         host_status: hosts[0].host_status,
         metadata: hosts[0].metadata,
+        query_strategy_version: MetadataQueryStrategyVersions.VERSION_2,
       };
       const packagePolicy = docGenerator.generatePolicyPackagePolicy();
       packagePolicy.id = hosts[0].metadata.Endpoint.policy.applied.id;

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts

@@ -9,12 +9,64 @@ import {
   SavedObjectsServiceStart,
   SavedObjectsClientContract,
 } from 'src/core/server';
-import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server';
+import {
+  AgentService,
+  IngestManagerStartContract,
+  PackageService,
+} from '../../../ingest_manager/server';
 import { getPackagePolicyCreateCallback } from './ingest_integration';
 import { ManifestManager } from './services/artifacts';
+import { MetadataQueryStrategy } from './types';
+import { MetadataQueryStrategyVersions } from '../../common/endpoint/types';
+import {
+  metadataQueryStrategyV1,
+  metadataQueryStrategyV2,
+} from './routes/metadata/support/query_strategies';
+import { ElasticsearchAssetType } from '../../../ingest_manager/common/types/models';
+import { metadataTransformPrefix } from '../../common/endpoint/constants';
+
+export interface MetadataService {
+  queryStrategy(
+    savedObjectsClient: SavedObjectsClientContract,
+    version?: MetadataQueryStrategyVersions
+  ): Promise<MetadataQueryStrategy>;
+}
+
+export const createMetadataService = (packageService: PackageService): MetadataService => {
+  return {
+    async queryStrategy(
+      savedObjectsClient: SavedObjectsClientContract,
+      version?: MetadataQueryStrategyVersions
+    ): Promise<MetadataQueryStrategy> {
+      if (version === MetadataQueryStrategyVersions.VERSION_1) {
+        return metadataQueryStrategyV1();
+      }
+      if (!packageService) {
+        throw new Error('package service is uninitialized');
+      }
+
+      if (version === MetadataQueryStrategyVersions.VERSION_2 || !version) {
+        const assets = await packageService.getInstalledEsAssetReferences(
+          savedObjectsClient,
+          'endpoint'
+        );
+        const expectedTransformAssets = assets.filter(
+          (ref) =>
+            ref.type === ElasticsearchAssetType.transform &&
+            ref.id.startsWith(metadataTransformPrefix)
+        );
+        if (expectedTransformAssets && expectedTransformAssets.length === 1) {
+          return metadataQueryStrategyV2();
+        }
+        return metadataQueryStrategyV1();
+      }
+      return metadataQueryStrategyV1();
+    },
+  };
+};
 
 export type EndpointAppContextServiceStartContract = Partial<
-  Pick<IngestManagerStartContract, 'agentService'>
+  Pick<IngestManagerStartContract, 'agentService' | 'packageService'>
 > & {
   logger: Logger;
   manifestManager?: ManifestManager;
@@ -30,11 +82,13 @@ export class EndpointAppContextService {
   private agentService: AgentService | undefined;
   private manifestManager: ManifestManager | undefined;
   private savedObjectsStart: SavedObjectsServiceStart | undefined;
+  private metadataService: MetadataService | undefined;
 
   public start(dependencies: EndpointAppContextServiceStartContract) {
     this.agentService = dependencies.agentService;
     this.manifestManager = dependencies.manifestManager;
     this.savedObjectsStart = dependencies.savedObjectsStart;
+    this.metadataService = createMetadataService(dependencies.packageService!);
 
     if (this.manifestManager && dependencies.registerIngestCallback) {
       dependencies.registerIngestCallback(
@@ -50,6 +104,10 @@ export class EndpointAppContextService {
     return this.agentService;
   }
 
+  public getMetadataService(): MetadataService | undefined {
+    return this.metadataService;
+  }
+
   public getManifestManager(): ManifestManager | undefined {
     return this.manifestManager;
   }

x-pack/plugins/security_solution/server/endpoint/mocks.ts

@@ -11,6 +11,7 @@ import {
   AgentService,
   IngestManagerStartContract,
   ExternalCallback,
+  PackageService,
 } from '../../../ingest_manager/server';
 import { createPackagePolicyServiceMock } from '../../../ingest_manager/server/mocks';
 import { createMockConfig } from '../lib/detection_engine/routes/__mocks__';
@@ -58,6 +59,7 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked<
 > => {
   return {
     agentService: createMockAgentService(),
+    packageService: createMockPackageService(),
     logger: loggingSystemMock.create().get('mock_endpoint_app_context'),
     savedObjectsStart: savedObjectsServiceMock.createStartContract(),
     manifestManager: getManifestManagerMock(),
@@ -68,6 +70,16 @@ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked<
   };
 };
 
+/**
+ * Create mock PackageService
+ */
+
+export const createMockPackageService = (): jest.Mocked<PackageService> => {
+  return {
+    getInstalledEsAssetReferences: jest.fn(),
+  };
+};
+
 /**
  * Creates a mock AgentService
  */
@@ -95,6 +107,7 @@ export const createMockIngestManagerStartContract = (
       getESIndexPattern: jest.fn().mockResolvedValue(indexPattern),
     },
     agentService: createMockAgentService(),
+    packageService: createMockPackageService(),
     registerExternalCallback: jest.fn((...args: ExternalCallback) => {}),
     packagePolicyService: createPackagePolicyServiceMock(),
   };