[Metrics UI] Replace Snapshot API with Metrics API (elastic#76253)

- Remove server/lib/snapshot
- Replace backend for /api/infra/snapshot with data from Metrics API
- Fixing tests with updates to the snapshot node

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/infra/common/http_api/index.ts

@@ -10,3 +10,4 @@ export * from './log_entries';
 export * from './metrics_explorer';
 export * from './metrics_api';
 export * from './log_alerts';
+export * from './snapshot_api';

x-pack/plugins/infra/common/http_api/metrics_api.ts

@@ -33,7 +33,6 @@ export const MetricsAPIRequestRT = rt.intersection([
     afterKey: rt.union([rt.null, afterKeyObjectRT]),
     limit: rt.union([rt.number, rt.null, rt.undefined]),
     filters: rt.array(rt.object),
-    forceInterval: rt.boolean,
     dropLastBucket: rt.boolean,
     alignDataToEnd: rt.boolean,
   }),
@@ -59,7 +58,10 @@ export const MetricsAPIRowRT = rt.intersection([
   rt.type({
     timestamp: rt.number,
   }),
-  rt.record(rt.string, rt.union([rt.string, rt.number, rt.null, rt.undefined])),
+  rt.record(
+    rt.string,
+    rt.union([rt.string, rt.number, rt.null, rt.undefined, rt.array(rt.object)])
+  ),
 ]);
 
 export const MetricsAPISeriesRT = rt.intersection([

x-pack/plugins/infra/common/http_api/metrics_explorer.ts

@@ -89,7 +89,10 @@ export const metricsExplorerRowRT = rt.intersection([
   rt.type({
     timestamp: rt.number,
   }),
-  rt.record(rt.string, rt.union([rt.string, rt.number, rt.null, rt.undefined])),
+  rt.record(
+    rt.string,
+    rt.union([rt.string, rt.number, rt.null, rt.undefined, rt.array(rt.object)])
+  ),
 ]);
 
 export const metricsExplorerSeriesRT = rt.intersection([

x-pack/plugins/infra/common/http_api/snapshot_api.ts

@@ -6,7 +6,7 @@
 
 import * as rt from 'io-ts';
 import { SnapshotMetricTypeRT, ItemTypeRT } from '../inventory_models/types';
-import { metricsExplorerSeriesRT } from './metrics_explorer';
+import { MetricsAPISeriesRT } from './metrics_api';
 
 export const SnapshotNodePathRT = rt.intersection([
   rt.type({
@@ -22,7 +22,7 @@ const SnapshotNodeMetricOptionalRT = rt.partial({
   value: rt.union([rt.number, rt.null]),
   avg: rt.union([rt.number, rt.null]),
   max: rt.union([rt.number, rt.null]),
-  timeseries: metricsExplorerSeriesRT,
+  timeseries: MetricsAPISeriesRT,
 });
 
 const SnapshotNodeMetricRequiredRT = rt.type({
@@ -36,6 +36,7 @@ export const SnapshotNodeMetricRT = rt.intersection([
 export const SnapshotNodeRT = rt.type({
   metrics: rt.array(SnapshotNodeMetricRT),
   path: rt.array(SnapshotNodePathRT),
+  name: rt.string,
 });
 
 export const SnapshotNodeResponseRT = rt.type({

x-pack/plugins/infra/common/inventory_models/types.ts

@@ -281,6 +281,10 @@ export const ESSumBucketAggRT = rt.type({
   }),
 });
 
+export const ESTopHitsAggRT = rt.type({
+  top_hits: rt.object,
+});
+
 interface SnapshotTermsWithAggregation {
   terms: { field: string };
   aggregations: MetricsUIAggregation;
@@ -304,6 +308,7 @@ export const ESAggregationRT = rt.union([
   ESSumBucketAggRT,
   ESTermsWithAggregationRT,
   ESCaridnalityAggRT,
+  ESTopHitsAggRT,
 ]);
 
 export const MetricsUIAggregationRT = rt.record(rt.string, ESAggregationRT);

x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/conditional_tooltip.test.tsx

@@ -88,6 +88,7 @@ describe('ConditionalToolTip', () => {
     mockedUseSnapshot.mockReturnValue({
       nodes: [
         {
+          name: 'host-01',
           path: [{ label: 'host-01', value: 'host-01', ip: '192.168.1.10' }],
           metrics: [
             { name: 'cpu', value: 0.1, avg: 0.4, max: 0.7 },

x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/calculate_bounds_from_nodes.test.ts

@@ -7,6 +7,7 @@ import { calculateBoundsFromNodes } from './calculate_bounds_from_nodes';
 import { SnapshotNode } from '../../../../../common/http_api/snapshot_api';
 const nodes: SnapshotNode[] = [
   {
+    name: 'host-01',
     path: [{ value: 'host-01', label: 'host-01' }],
     metrics: [
       {
@@ -18,6 +19,7 @@ const nodes: SnapshotNode[] = [
     ],
   },
   {
+    name: 'host-02',
     path: [{ value: 'host-02', label: 'host-02' }],
     metrics: [
       {

x-pack/plugins/infra/public/pages/metrics/inventory_view/lib/sort_nodes.test.ts

@@ -9,6 +9,7 @@ import { SnapshotNode } from '../../../../../common/http_api/snapshot_api';
 
 const nodes: SnapshotNode[] = [
   {
+    name: 'host-01',
     path: [{ value: 'host-01', label: 'host-01' }],
     metrics: [
       {
@@ -20,6 +21,7 @@ const nodes: SnapshotNode[] = [
     ],
   },
   {
+    name: 'host-02',
     path: [{ value: 'host-02', label: 'host-02' }],
     metrics: [
       {

x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/evaluate_condition.ts

@@ -16,12 +16,11 @@ import {
 } from '../../adapters/framework/adapter_types';
 import { Comparator, InventoryMetricConditions } from './types';
 import { AlertServices } from '../../../../../alerts/server';
-import { InfraSnapshot } from '../../snapshot';
-import { parseFilterQuery } from '../../../utils/serialized_query';
 import { InventoryItemType, SnapshotMetricType } from '../../../../common/inventory_models/types';
-import { InfraTimerangeInput } from '../../../../common/http_api/snapshot_api';
-import { InfraSourceConfiguration } from '../../sources';
+import { InfraTimerangeInput, SnapshotRequest } from '../../../../common/http_api/snapshot_api';
+import { InfraSource } from '../../sources';
 import { UNGROUPED_FACTORY_KEY } from '../common/utils';
+import { getNodes } from '../../../routes/snapshot/lib/get_nodes';
 
 type ConditionResult = InventoryMetricConditions & {
   shouldFire: boolean[];
@@ -33,7 +32,7 @@ type ConditionResult = InventoryMetricConditions & {
 export const evaluateCondition = async (
   condition: InventoryMetricConditions,
   nodeType: InventoryItemType,
-  sourceConfiguration: InfraSourceConfiguration,
+  source: InfraSource,
   callCluster: AlertServices['callCluster'],
   filterQuery?: string,
   lookbackSize?: number
@@ -55,7 +54,7 @@ export const evaluateCondition = async (
     nodeType,
     metric,
     timerange,
-    sourceConfiguration,
+    source,
     filterQuery,
     customMetric
   );
@@ -94,30 +93,29 @@ const getData = async (
   nodeType: InventoryItemType,
   metric: SnapshotMetricType,
   timerange: InfraTimerangeInput,
-  sourceConfiguration: InfraSourceConfiguration,
+  source: InfraSource,
   filterQuery?: string,
   customMetric?: SnapshotCustomMetricInput
 ) => {
-  const snapshot = new InfraSnapshot();
-  const esClient = <Hit = {}, Aggregation = undefined>(
+  const client = <Hit = {}, Aggregation = undefined>(
     options: CallWithRequestParams
   ): Promise<InfraDatabaseSearchResponse<Hit, Aggregation>> => callCluster('search', options);
 
   const metrics = [
     metric === 'custom' ? (customMetric as SnapshotCustomMetricInput) : { type: metric },
   ];
 
-  const options = {
-    filterQuery: parseFilterQuery(filterQuery),
+  const snapshotRequest: SnapshotRequest = {
+    filterQuery,
     nodeType,
     groupBy: [],
-    sourceConfiguration,
+    sourceId: 'default',
     metrics,
     timerange,
     includeTimeseries: Boolean(timerange.lookbackSize),
   };
   try {
-    const { nodes } = await snapshot.getNodes(esClient, options);
+    const { nodes } = await getNodes(client, snapshotRequest, source);
 
     if (!nodes.length) return { [UNGROUPED_FACTORY_KEY]: null }; // No Data state
 

x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/inventory_metric_threshold_executor.ts

@@ -50,9 +50,7 @@ export const createInventoryMetricThresholdExecutor = (libs: InfraBackendLibs) =
   );
 
   const results = await Promise.all(
-    criteria.map((c) =>
-      evaluateCondition(c, nodeType, source.configuration, services.callCluster, filterQuery)
-    )
+    criteria.map((c) => evaluateCondition(c, nodeType, source, services.callCluster, filterQuery))
   );
 
   const inventoryItems = Object.keys(first(results)!);

x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/preview_inventory_metric_threshold_alert.ts

@@ -26,15 +26,15 @@ interface InventoryMetricThresholdParams {
 interface PreviewInventoryMetricThresholdAlertParams {
   callCluster: ILegacyScopedClusterClient['callAsCurrentUser'];
   params: InventoryMetricThresholdParams;
-  config: InfraSource['configuration'];
+  source: InfraSource;
   lookback: Unit;
   alertInterval: string;
 }
 
 export const previewInventoryMetricThresholdAlert = async ({
   callCluster,
   params,
-  config,
+  source,
   lookback,
   alertInterval,
 }: PreviewInventoryMetricThresholdAlertParams) => {
@@ -55,7 +55,7 @@ export const previewInventoryMetricThresholdAlert = async ({
   try {
     const results = await Promise.all(
       criteria.map((c) =>
-        evaluateCondition(c, nodeType, config, callCluster, filterQuery, lookbackSize)
+        evaluateCondition(c, nodeType, source, callCluster, filterQuery, lookbackSize)
       )
     );
 

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts

@@ -8,8 +8,8 @@ import { networkTraffic } from '../../../../../common/inventory_models/shared/me
 import { MetricExpressionParams, Aggregators } from '../types';
 import { getIntervalInSeconds } from '../../../../utils/get_interval_in_seconds';
 import { roundTimestamp } from '../../../../utils/round_timestamp';
-import { getDateHistogramOffset } from '../../../snapshot/query_helpers';
 import { createPercentileAggregation } from './create_percentile_aggregation';
+import { calculateDateHistogramOffset } from '../../../metrics/lib/calculate_date_histogram_offset';
 
 const MINIMUM_BUCKETS = 5;
 
@@ -46,7 +46,7 @@ export const getElasticsearchMetricQuery = (
     timeUnit
   );
 
-  const offset = getDateHistogramOffset(from, interval);
+  const offset = calculateDateHistogramOffset({ from, to, interval, field: timefield });
 
   const aggregations =
     aggType === Aggregators.COUNT

x-pack/plugins/infra/server/lib/infra_types.ts

@@ -8,7 +8,6 @@ import { InfraSourceConfiguration } from '../../common/graphql/types';
 import { InfraFieldsDomain } from './domains/fields_domain';
 import { InfraLogEntriesDomain } from './domains/log_entries_domain';
 import { InfraMetricsDomain } from './domains/metrics_domain';
-import { InfraSnapshot } from './snapshot';
 import { InfraSources } from './sources';
 import { InfraSourceStatus } from './source_status';
 import { InfraConfig } from '../plugin';
@@ -30,7 +29,6 @@ export interface InfraDomainLibs {
 export interface InfraBackendLibs extends InfraDomainLibs {
   configuration: InfraConfig;
   framework: KibanaFramework;
-  snapshot: InfraSnapshot;
   sources: InfraSources;
   sourceStatus: InfraSourceStatus;
 }

x-pack/plugins/infra/server/lib/metrics/lib/convert_histogram_buckets_to_timeseries.ts

@@ -5,6 +5,7 @@
  */
 
 import { get, values, first } from 'lodash';
+import * as rt from 'io-ts';
 import {
   MetricsAPIRequest,
   MetricsAPISeries,
@@ -13,15 +14,20 @@ import {
 } from '../../../../common/http_api/metrics_api';
 import {
   HistogramBucket,
-  MetricValueType,
   BasicMetricValueRT,
   NormalizedMetricValueRT,
   PercentilesTypeRT,
   PercentilesKeyedTypeRT,
+  TopHitsTypeRT,
+  MetricValueTypeRT,
 } from '../types';
+
 const BASE_COLUMNS = [{ name: 'timestamp', type: 'date' }] as MetricsAPIColumn[];
 
-const getValue = (valueObject: string | number | MetricValueType) => {
+const ValueObjectTypeRT = rt.union([rt.string, rt.number, MetricValueTypeRT]);
+type ValueObjectType = rt.TypeOf<typeof ValueObjectTypeRT>;
+
+const getValue = (valueObject: ValueObjectType) => {
   if (NormalizedMetricValueRT.is(valueObject)) {
     return valueObject.normalized_value || valueObject.value;
   }
@@ -50,6 +56,10 @@ const getValue = (valueObject: string | number | MetricValueType) => {
     return valueObject.value;
   }
 
+  if (TopHitsTypeRT.is(valueObject)) {
+    return valueObject.hits.hits.map((hit) => hit._source);
+  }
+
   return null;
 };
 
@@ -61,8 +71,8 @@ const convertBucketsToRows = (
     const ids = options.metrics.map((metric) => metric.id);
     const metrics = ids.reduce((acc, id) => {
       const valueObject = get(bucket, [id]);
-      return { ...acc, [id]: getValue(valueObject) };
-    }, {} as Record<string, number | null>);
+      return { ...acc, [id]: ValueObjectTypeRT.is(valueObject) ? getValue(valueObject) : null };
+    }, {} as Record<string, number | null | object[]>);
     return { timestamp: bucket.key as number, ...metrics };
   });
 };

x-pack/plugins/infra/server/lib/metrics/lib/create_aggregations.ts

@@ -33,7 +33,7 @@ export const createAggregations = (options: MetricsAPIRequest) => {
         composite: {
           size: limit,
           sources: options.groupBy.map((field, index) => ({
-            [`groupBy${index}`]: { terms: { field, order: 'asc' } },
+            [`groupBy${index}`]: { terms: { field } },
           })),
         },
         aggs: histogramAggregation,

x-pack/plugins/infra/server/lib/metrics/types.ts

@@ -25,17 +25,51 @@ export const PercentilesKeyedTypeRT = rt.type({
   values: rt.array(rt.type({ key: rt.string, value: NumberOrNullRT })),
 });
 
+export const TopHitsTypeRT = rt.type({
+  hits: rt.type({
+    total: rt.type({
+      value: rt.number,
+      relation: rt.string,
+    }),
+    hits: rt.array(
+      rt.intersection([
+        rt.type({
+          _index: rt.string,
+          _id: rt.string,
+          _score: NumberOrNullRT,
+          _source: rt.object,
+        }),
+        rt.partial({
+          sort: rt.array(rt.union([rt.string, rt.number])),
+          max_score: NumberOrNullRT,
+        }),
+      ])
+    ),
+  }),
+});
+
 export const MetricValueTypeRT = rt.union([
   BasicMetricValueRT,
   NormalizedMetricValueRT,
   PercentilesTypeRT,
   PercentilesKeyedTypeRT,
+  TopHitsTypeRT,
 ]);
 export type MetricValueType = rt.TypeOf<typeof MetricValueTypeRT>;
 
+export const TermsWithMetrics = rt.intersection([
+  rt.type({
+    buckets: rt.array(rt.record(rt.string, rt.union([rt.number, rt.string, MetricValueTypeRT]))),
+  }),
+  rt.partial({
+    sum_other_doc_count: rt.number,
+    doc_count_error_upper_bound: rt.number,
+  }),
+]);
+
 export const HistogramBucketRT = rt.record(
   rt.string,
-  rt.union([rt.number, rt.string, MetricValueTypeRT])
+  rt.union([rt.number, rt.string, MetricValueTypeRT, TermsWithMetrics])
 );
 
 export const HistogramResponseRT = rt.type({