[7.x] [Fleet] Set default settings in component template instead of the index template (#111197)

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { loggerMock } from '@kbn/logging/mocks';
+import type { Logger } from 'src/core/server';
+
+import { appContextService } from '../../../app_context';
+
+import { buildDefaultSettings } from './default_settings';
+
+jest.mock('../../../app_context');
+
+const mockedAppContextService = appContextService as jest.Mocked<typeof appContextService>;
+let mockedLogger: jest.Mocked<Logger>;
+describe('buildDefaultSettings', () => {
+  beforeEach(() => {
+    mockedLogger = loggerMock.create();
+    mockedAppContextService.getLogger.mockReturnValue(mockedLogger);
+  });
+
+  it('should generate default settings', () => {
+    const settings = buildDefaultSettings({
+      templateName: 'test_template',
+      packageName: 'test_package',
+      type: 'logs',
+      fields: [
+        {
+          name: 'field1Keyword',
+          type: 'keyword',
+        },
+        {
+          name: 'field2Boolean',
+          type: 'boolean',
+        },
+      ],
+    });
+
+    expect(settings).toMatchInlineSnapshot(`
+      Object {
+        "index": Object {
+          "codec": "best_compression",
+          "lifecycle": Object {
+            "name": "logs",
+          },
+          "mapping": Object {
+            "total_fields": Object {
+              "limit": "10000",
+            },
+          },
+          "number_of_routing_shards": "30",
+          "number_of_shards": "1",
+          "query": Object {
+            "default_field": Array [
+              "field1Keyword",
+            ],
+          },
+          "refresh_interval": "5s",
+        },
+      }
+    `);
+  });
+
+  it('should log a warning if there is too many default fields', () => {
+    const fields = [];
+    for (let i = 0; i < 20000; i++) {
+      fields.push({ name: `field${i}`, type: 'keyword' });
+    }
+    buildDefaultSettings({
+      type: 'logs',
+      templateName: 'test_template',
+      packageName: 'test_package',
+      fields,
+    });
+
+    expect(mockedLogger.warn).toBeCalledWith(
+      'large amount of default fields detected for index template test_template in package test_package, applying the first 1024 fields'
+    );
+  });
+});

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { appContextService } from '../../../app_context';
+import type { Field, Fields } from '../../fields/field';
+
+const QUERY_DEFAULT_FIELD_TYPES = ['keyword', 'text'];
+const QUERY_DEFAULT_FIELD_LIMIT = 1024;
+
+const flattenFieldsToNameAndType = (
+  fields: Fields,
+  path: string = ''
+): Array<Pick<Field, 'name' | 'type'>> => {
+  let newFields: Array<Pick<Field, 'name' | 'type'>> = [];
+  fields.forEach((field) => {
+    const fieldName = path ? `${path}.${field.name}` : field.name;
+    newFields.push({
+      name: fieldName,
+      type: field.type,
+    });
+    if (field.fields && field.fields.length) {
+      newFields = newFields.concat(flattenFieldsToNameAndType(field.fields, fieldName));
+    }
+  });
+  return newFields;
+};
+
+export function buildDefaultSettings({
+  templateName,
+  packageName,
+  fields,
+  ilmPolicy,
+  type,
+}: {
+  type: string;
+  templateName: string;
+  packageName: string;
+  ilmPolicy?: string | undefined;
+  fields: Field[];
+}) {
+  const logger = appContextService.getLogger();
+  // Find all field names to set `index.query.default_field` to, which will be
+  // the first 1024 keyword or text fields
+  const defaultFields = flattenFieldsToNameAndType(fields).filter(
+    (field) => field.type && QUERY_DEFAULT_FIELD_TYPES.includes(field.type)
+  );
+  if (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT) {
+    logger.warn(
+      `large amount of default fields detected for index template ${templateName} in package ${packageName}, applying the first ${QUERY_DEFAULT_FIELD_LIMIT} fields`
+    );
+  }
+  const defaultFieldNames = (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT
+    ? defaultFields.slice(0, QUERY_DEFAULT_FIELD_LIMIT)
+    : defaultFields
+  ).map((field) => field.name);
+
+  return {
+    index: {
+      // ILM Policy must be added here, for now point to the default global ILM policy name
+      lifecycle: {
+        name: ilmPolicy ? ilmPolicy : type,
+      },
+      // What should be our default for the compression?
+      codec: 'best_compression',
+      mapping: {
+        total_fields: {
+          limit: '10000',
+        },
+      },
+      // This is the default from Beats? So far seems to be a good value
+      refresh_interval: '5s',
+      // Default in the stack now, still good to have it in
+      number_of_shards: '1',
+      // We are setting 30 because it can be devided by several numbers. Useful when shrinking.
+      number_of_routing_shards: '30',
+
+      // All the default fields which should be queried have to be added here.
+      // So far we add all keyword and text fields here if there are any, otherwise
+      // this setting is skipped.
+      ...(defaultFieldNames.length
+        ? {
+            query: {
+              default_field: defaultFieldNames,
+            },
+          }
+        : {}),
+    },
+  };
+}

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { merge } from 'lodash';
 import Boom from '@hapi/boom';
 import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
 
@@ -14,6 +15,7 @@ import type {
   IndexTemplateEntry,
   RegistryElasticsearch,
   InstallablePackage,
+  IndexTemplate,
 } from '../../../../types';
 import { loadFieldsFromYaml, processFields } from '../../fields/field';
 import type { Field } from '../../fields/field';
@@ -32,6 +34,7 @@ import {
   getTemplate,
   getTemplatePriority,
 } from './template';
+import { buildDefaultSettings } from './default_settings';
 
 export const installTemplates = async (
   installablePackage: InstallablePackage,
@@ -210,8 +213,9 @@ function buildComponentTemplates(params: {
   templateName: string;
   registryElasticsearch: RegistryElasticsearch | undefined;
   packageName: string;
+  defaultSettings: IndexTemplate['template']['settings'];
 }) {
-  const { templateName, registryElasticsearch, packageName } = params;
+  const { templateName, registryElasticsearch, packageName, defaultSettings } = params;
   const mappingsTemplateName = `${templateName}${mappingsSuffix}`;
   const settingsTemplateName = `${templateName}${settingsSuffix}`;
   const userSettingsTemplateName = `${templateName}${userSettingsSuffix}`;
@@ -228,14 +232,12 @@ function buildComponentTemplates(params: {
     };
   }
 
-  if (registryElasticsearch && registryElasticsearch['index_template.settings']) {
-    templatesMap[settingsTemplateName] = {
-      template: {
-        settings: registryElasticsearch['index_template.settings'],
-      },
-      _meta,
-    };
-  }
+  templatesMap[settingsTemplateName] = {
+    template: {
+      settings: merge(defaultSettings, registryElasticsearch?.['index_template.settings'] ?? {}),
+    },
+    _meta,
+  };
 
   // return empty/stub template
   templatesMap[userSettingsTemplateName] = {
@@ -253,9 +255,15 @@ async function installDataStreamComponentTemplates(params: {
   registryElasticsearch: RegistryElasticsearch | undefined;
   esClient: ElasticsearchClient;
   packageName: string;
+  defaultSettings: IndexTemplate['template']['settings'];
 }) {
-  const { templateName, registryElasticsearch, esClient, packageName } = params;
-  const templates = buildComponentTemplates({ templateName, registryElasticsearch, packageName });
+  const { templateName, registryElasticsearch, esClient, packageName, defaultSettings } = params;
+  const templates = buildComponentTemplates({
+    templateName,
+    registryElasticsearch,
+    packageName,
+    defaultSettings,
+  });
   const templateNames = Object.keys(templates);
   const templateEntries = Object.entries(templates);
 
@@ -362,11 +370,20 @@ export async function installTemplate({
     await esClient.indices.putIndexTemplate(updateIndexTemplateParams, { ignore: [404] });
   }
 
+  const defaultSettings = buildDefaultSettings({
+    templateName,
+    packageName,
+    fields,
+    type: dataStream.type,
+    ilmPolicy: dataStream.ilm_policy,
+  });
+
   const composedOfTemplates = await installDataStreamComponentTemplates({
     templateName,
     registryElasticsearch: dataStream.elasticsearch,
     esClient,
     packageName,
+    defaultSettings,
   });
 
   const template = getTemplate({
@@ -378,7 +395,6 @@ export async function installTemplate({
     packageName,
     composedOfTemplates,
     templatePriority,
-    ilmPolicy: dataStream.ilm_policy,
     hidden: dataStream.hidden,
   });