[Ingest] Allow aggent to send metadata compliant with ECS (#64452)

x-pack/plugins/ingest_manager/common/types/models/agent.ts

@@ -60,6 +60,11 @@ export interface AgentEvent {
 
 export interface AgentEventSOAttributes extends AgentEvent, SavedObjectAttributes {}
 
+type MetadataValue = string | AgentMetadata;
+
+export interface AgentMetadata {
+  [x: string]: MetadataValue;
+}
 interface AgentBase {
   type: AgentType;
   active: boolean;
@@ -72,19 +77,17 @@ interface AgentBase {
   config_revision?: number | null;
   config_newest_revision?: number;
   last_checkin?: string;
+  user_provided_metadata: AgentMetadata;
+  local_metadata: AgentMetadata;
 }
 
 export interface Agent extends AgentBase {
   id: string;
   current_error_events: AgentEvent[];
-  user_provided_metadata: Record<string, string>;
-  local_metadata: Record<string, string>;
   access_api_key?: string;
   status?: string;
 }
 
 export interface AgentSOAttributes extends AgentBase, SavedObjectAttributes {
-  user_provided_metadata: string;
-  local_metadata: string;
   current_error_events?: string;
 }

x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/search_bar.tsx

@@ -92,7 +92,6 @@ function useSuggestions(fieldPrefix: string, search: string) {
       const res = (await data.indexPatterns.getFieldsForWildcard({
         pattern: INDEX_NAME,
       })) as IFieldType[];
-
       if (!data || !data.autocomplete) {
         throw new Error('Missing data plugin');
       }

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/helper.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { AgentMetadata } from '../../../../types';
+
+export function flattenMetadata(metadata: AgentMetadata) {
+  return Object.entries(metadata).reduce((acc, [key, value]) => {
+    if (typeof value === 'string') {
+      acc[key] = value;
+
+      return acc;
+    }
+
+    Object.entries(flattenMetadata(value)).forEach(([flattenedKey, flattenedValue]) => {
+      acc[`${key}.${flattenedKey}`] = flattenedValue;
+    });
+
+    return acc;
+  }, {} as { [k: string]: string });
+}
+export function unflattenMetadata(flattened: { [k: string]: string }) {
+  const metadata: AgentMetadata = {};
+
+  Object.entries(flattened).forEach(([flattenedKey, flattenedValue]) => {
+    const keyParts = flattenedKey.split('.');
+    const lastKey = keyParts.pop();
+
+    if (!lastKey) {
+      throw new Error('Invalid metadata');
+    }
+
+    let metadataPart = metadata;
+    keyParts.forEach(keyPart => {
+      if (!metadataPart[keyPart]) {
+        metadataPart[keyPart] = {};
+      }
+
+      metadataPart = metadataPart[keyPart] as AgentMetadata;
+    });
+    metadataPart[lastKey] = flattenedValue;
+  });
+
+  return metadata;
+}

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_flyout.tsx

@@ -17,11 +17,13 @@ import {
 } from '@elastic/eui';
 import { MetadataForm } from './metadata_form';
 import { Agent } from '../../../../types';
+import { flattenMetadata } from './helper';
 
 interface Props {
   agent: Agent;
   flyout: { hide: () => void };
 }
+
 export const AgentMetadataFlyout: React.FunctionComponent<Props> = ({ agent, flyout }) => {
   const mapMetadata = (obj: { [key: string]: string } | undefined) => {
     return Object.keys(obj || {}).map(key => ({
@@ -30,8 +32,8 @@ export const AgentMetadataFlyout: React.FunctionComponent<Props> = ({ agent, fly
     }));
   };
 
-  const localItems = mapMetadata(agent.local_metadata);
-  const userProvidedItems = mapMetadata(agent.user_provided_metadata);
+  const localItems = mapMetadata(flattenMetadata(agent.local_metadata));
+  const userProvidedItems = mapMetadata(flattenMetadata(agent.user_provided_metadata));
 
   return (
     <EuiFlyout onClose={() => flyout.hide()} size="s" aria-labelledby="flyoutTitle">

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/components/metadata_form.tsx

@@ -22,6 +22,7 @@ import { useAgentRefresh } from '../hooks';
 import { useInput, sendRequest } from '../../../../hooks';
 import { Agent } from '../../../../types';
 import { agentRouteService } from '../../../../services';
+import { flattenMetadata, unflattenMetadata } from './helper';
 
 function useAddMetadataForm(agent: Agent, done: () => void) {
   const refreshAgent = useAgentRefresh();
@@ -66,15 +67,17 @@ function useAddMetadataForm(agent: Agent, done: () => void) {
         isLoading: true,
       });
 
+      const metadata = unflattenMetadata({
+        ...flattenMetadata(agent.user_provided_metadata),
+        [keyInput.value]: valueInput.value,
+      });
+
       try {
         const { error } = await sendRequest({
           path: agentRouteService.getUpdatePath(agent.id),
           method: 'put',
           body: JSON.stringify({
-            user_provided_metadata: {
-              ...agent.user_provided_metadata,
-              [keyInput.value]: valueInput.value,
-            },
+            user_provided_metadata: metadata,
           }),
         });
 

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx

@@ -238,7 +238,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
 
   const columns = [
     {
-      field: 'local_metadata.host',
+      field: 'local_metadata.host.hostname',
       name: i18n.translate('xpack.ingestManager.agentList.hostColumnTitle', {
         defaultMessage: 'Host',
       }),

x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts

@@ -8,6 +8,7 @@ export {
   entries,
   // Object types
   Agent,
+  AgentMetadata,
   AgentConfig,
   NewAgentConfig,
   AgentEvent,

x-pack/plugins/ingest_manager/server/saved_objects.ts

@@ -56,8 +56,8 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
         enrolled_at: { type: 'date' },
         access_api_key_id: { type: 'keyword' },
         version: { type: 'keyword' },
-        user_provided_metadata: { type: 'text' },
-        local_metadata: { type: 'text' },
+        user_provided_metadata: { type: 'flattened' },
+        local_metadata: { type: 'flattened' },
         config_id: { type: 'keyword' },
         last_updated: { type: 'date' },
         last_checkin: { type: 'date' },

x-pack/plugins/ingest_manager/server/services/agents/checkin.ts

@@ -11,6 +11,7 @@ import {
   AgentAction,
   AgentSOAttributes,
   AgentEventSOAttributes,
+  AgentMetadata,
 } from '../../types';
 
 import { agentConfigService } from '../agent_config';
@@ -28,7 +29,7 @@ export async function agentCheckin(
   const updateData: {
     last_checkin: string;
     default_api_key?: string;
-    local_metadata?: string;
+    local_metadata?: AgentMetadata;
     current_error_events?: string;
   } = {
     last_checkin: new Date().toISOString(),

x-pack/plugins/ingest_manager/server/services/agents/crud.ts

@@ -103,7 +103,7 @@ export async function updateAgent(
   }
 ) {
   await soClient.update<AgentSOAttributes>(AGENT_SAVED_OBJECT_TYPE, agentId, {
-    user_provided_metadata: JSON.stringify(data.userProvidedMetatada),
+    user_provided_metadata: data.userProvidedMetatada,
   });
 }
 

x-pack/plugins/ingest_manager/server/services/agents/enroll.ts

@@ -32,8 +32,8 @@ export async function enroll(
     config_id: configId,
     type,
     enrolled_at: enrolledAt,
-    user_provided_metadata: JSON.stringify(metadata?.userProvided ?? {}),
-    local_metadata: JSON.stringify(metadata?.local ?? {}),
+    user_provided_metadata: metadata?.userProvided ?? {},
+    local_metadata: metadata?.local ?? {},
     current_error_events: undefined,
     access_api_key_id: undefined,
     last_checkin: undefined,

x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts

@@ -19,8 +19,8 @@ export function savedObjectToAgent(so: SavedObject<AgentSOAttributes>): Agent {
     current_error_events: so.attributes.current_error_events
       ? JSON.parse(so.attributes.current_error_events)
       : [],
-    local_metadata: JSON.parse(so.attributes.local_metadata),
-    user_provided_metadata: JSON.parse(so.attributes.user_provided_metadata),
+    local_metadata: so.attributes.local_metadata,
+    user_provided_metadata: so.attributes.user_provided_metadata,
     access_api_key: undefined,
     status: undefined,
   };

x-pack/plugins/ingest_manager/server/services/agents/status.test.ts

@@ -18,8 +18,8 @@ describe('Agent status service', () => {
       type: AGENT_TYPE_PERMANENT,
       attributes: {
         active: false,
-        local_metadata: '{}',
-        user_provided_metadata: '{}',
+        local_metadata: {},
+        user_provided_metadata: {},
       },
     } as SavedObject<AgentSOAttributes>);
     const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
@@ -33,8 +33,8 @@ describe('Agent status service', () => {
       type: AGENT_TYPE_PERMANENT,
       attributes: {
         active: true,
-        local_metadata: '{}',
-        user_provided_metadata: '{}',
+        local_metadata: {},
+        user_provided_metadata: {},
       },
     } as SavedObject<AgentSOAttributes>);
     const status = await getAgentStatusById(mockSavedObjectsClient, 'id');

x-pack/plugins/ingest_manager/server/types/index.tsx

@@ -8,6 +8,7 @@ import { ScopedClusterClient } from 'src/core/server';
 export {
   // Object types
   Agent,
+  AgentMetadata,
   AgentSOAttributes,
   AgentStatus,
   AgentType,

x-pack/test/functional/es_archives/fleet/agents/data.json

@@ -11,8 +11,8 @@
         "shared_id": "agent1_filebeat",
         "config_id": "1",
         "type": "PERMANENT",
-        "local_metadata": "{}",
-        "user_provided_metadata": "{}"
+        "local_metadata": {},
+        "user_provided_metadata": {}
       }
     }
   }
@@ -30,8 +30,8 @@
         "active": true,
         "shared_id": "agent2_filebeat",
         "type": "PERMANENT",
-        "local_metadata": "{}",
-        "user_provided_metadata": "{}"
+        "local_metadata": {},
+        "user_provided_metadata": {}
       }
     }
   }
@@ -49,8 +49,8 @@
         "active": true,
         "shared_id": "agent3_metricbeat",
         "type": "PERMANENT",
-        "local_metadata": "{}",
-        "user_provided_metadata": "{}"
+        "local_metadata": {},
+        "user_provided_metadata": {}
       }
     }
   }
@@ -68,8 +68,8 @@
         "active": true,
         "shared_id": "agent4_metricbeat",
         "type": "PERMANENT",
-        "local_metadata": "{}",
-        "user_provided_metadata": "{}"
+        "local_metadata": {},
+        "user_provided_metadata": {}
       }
     }
   }

x-pack/test/functional/es_archives/fleet/agents/mappings.json

@@ -227,7 +227,7 @@
               "type": "date"
             },
             "local_metadata": {
-              "type": "text"
+              "type": "flattened"
             },
             "shared_id": {
               "type": "keyword"
@@ -239,7 +239,7 @@
               "type": "date"
             },
             "user_provided_metadata": {
-              "type": "text"
+              "type": "flattened"
             },
             "version": {
               "type": "keyword"