From 62fa64fa6ab2e5566dbbf051d83d64e099ac143d Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 7 Sep 2021 12:09:08 -0400 Subject: [PATCH] [Fleet] Set default settings in component template instead of the index template (#111197) --- .../__snapshots__/template.test.ts.snap | 98 +------------------ .../template/default_settings.test.ts | 83 ++++++++++++++++ .../template/default_settings.ts | 93 ++++++++++++++++++ .../epm/elasticsearch/template/install.ts | 40 +++++--- .../epm/elasticsearch/template/template.ts | 55 +---------- .../epm/__snapshots__/install_by_upload.snap | 12 +++ .../apis/epm/install_by_upload.ts | 4 +- .../apis/epm/install_overrides.ts | 8 ++ .../apis/epm/install_remove_assets.ts | 4 + .../apis/epm/update_assets.ts | 24 ++++- 10 files changed, 257 insertions(+), 164 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index eff35a30ba2d6..0425573c5afaa 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -8,25 +8,7 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = ` ], "template": { "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "refresh_interval": "5s", - "number_of_shards": "1", - "number_of_routing_shards": "30", - "query": { - "default_field": [ - "long.nested.foo" - ] - } - } + "index": {} }, "mappings": { "dynamic_templates": [ @@ -123,30 +105,7 @@ exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` ], "template": { "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "refresh_interval": "5s", - "number_of_shards": "1", - "number_of_routing_shards": "30", - "query": { - "default_field": [ - "coredns.id", - "coredns.query.class", - "coredns.query.name", - "coredns.query.type", - "coredns.response.code", - "coredns.response.flags" - ] - } - } + "index": {} }, "mappings": { "dynamic_templates": [ @@ -239,58 +198,7 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` ], "template": { "settings": { - "index": { - "lifecycle": { - "name": "metrics" - }, - "codec": "best_compression", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "refresh_interval": "5s", - "number_of_shards": "1", - "number_of_routing_shards": "30", - "query": { - "default_field": [ - "system.diskio.name", - "system.diskio.serial_number", - "system.filesystem.device_name", - "system.filesystem.type", - "system.filesystem.mount_point", - "system.network.name", - "system.process.state", - "system.process.cmdline", - "system.process.cgroup.id", - "system.process.cgroup.path", - "system.process.cgroup.cpu.id", - "system.process.cgroup.cpu.path", - "system.process.cgroup.cpuacct.id", - "system.process.cgroup.cpuacct.path", - "system.process.cgroup.memory.id", - "system.process.cgroup.memory.path", - "system.process.cgroup.blkio.id", - "system.process.cgroup.blkio.path", - "system.raid.name", - "system.raid.status", - "system.raid.level", - "system.raid.sync_action", - "system.socket.remote.host", - "system.socket.remote.etld_plus_one", - "system.socket.remote.host_error", - "system.socket.process.cmdline", - "system.users.id", - "system.users.seat", - "system.users.path", - "system.users.type", - "system.users.service", - "system.users.state", - "system.users.scope", - "system.users.remote_host" - ] - } - } + "index": {} }, "mappings": { "dynamic_templates": [ diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts new file mode 100644 index 0000000000000..5e7a3b35c544a --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock } from '@kbn/logging/mocks'; +import type { Logger } from 'src/core/server'; + +import { appContextService } from '../../../app_context'; + +import { buildDefaultSettings } from './default_settings'; + +jest.mock('../../../app_context'); + +const mockedAppContextService = appContextService as jest.Mocked; +let mockedLogger: jest.Mocked; +describe('buildDefaultSettings', () => { + beforeEach(() => { + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); + }); + + it('should generate default settings', () => { + const settings = buildDefaultSettings({ + templateName: 'test_template', + packageName: 'test_package', + type: 'logs', + fields: [ + { + name: 'field1Keyword', + type: 'keyword', + }, + { + name: 'field2Boolean', + type: 'boolean', + }, + ], + }); + + expect(settings).toMatchInlineSnapshot(` + Object { + "index": Object { + "codec": "best_compression", + "lifecycle": Object { + "name": "logs", + }, + "mapping": Object { + "total_fields": Object { + "limit": "10000", + }, + }, + "number_of_routing_shards": "30", + "number_of_shards": "1", + "query": Object { + "default_field": Array [ + "field1Keyword", + ], + }, + "refresh_interval": "5s", + }, + } + `); + }); + + it('should log a warning if there is too many default fields', () => { + const fields = []; + for (let i = 0; i < 20000; i++) { + fields.push({ name: `field${i}`, type: 'keyword' }); + } + buildDefaultSettings({ + type: 'logs', + templateName: 'test_template', + packageName: 'test_package', + fields, + }); + + expect(mockedLogger.warn).toBeCalledWith( + 'large amount of default fields detected for index template test_template in package test_package, applying the first 1024 fields' + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts new file mode 100644 index 0000000000000..2dced977229e1 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { appContextService } from '../../../app_context'; +import type { Field, Fields } from '../../fields/field'; + +const QUERY_DEFAULT_FIELD_TYPES = ['keyword', 'text']; +const QUERY_DEFAULT_FIELD_LIMIT = 1024; + +const flattenFieldsToNameAndType = ( + fields: Fields, + path: string = '' +): Array> => { + let newFields: Array> = []; + fields.forEach((field) => { + const fieldName = path ? `${path}.${field.name}` : field.name; + newFields.push({ + name: fieldName, + type: field.type, + }); + if (field.fields && field.fields.length) { + newFields = newFields.concat(flattenFieldsToNameAndType(field.fields, fieldName)); + } + }); + return newFields; +}; + +export function buildDefaultSettings({ + templateName, + packageName, + fields, + ilmPolicy, + type, +}: { + type: string; + templateName: string; + packageName: string; + ilmPolicy?: string | undefined; + fields: Field[]; +}) { + const logger = appContextService.getLogger(); + // Find all field names to set `index.query.default_field` to, which will be + // the first 1024 keyword or text fields + const defaultFields = flattenFieldsToNameAndType(fields).filter( + (field) => field.type && QUERY_DEFAULT_FIELD_TYPES.includes(field.type) + ); + if (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT) { + logger.warn( + `large amount of default fields detected for index template ${templateName} in package ${packageName}, applying the first ${QUERY_DEFAULT_FIELD_LIMIT} fields` + ); + } + const defaultFieldNames = (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT + ? defaultFields.slice(0, QUERY_DEFAULT_FIELD_LIMIT) + : defaultFields + ).map((field) => field.name); + + return { + index: { + // ILM Policy must be added here, for now point to the default global ILM policy name + lifecycle: { + name: ilmPolicy ? ilmPolicy : type, + }, + // What should be our default for the compression? + codec: 'best_compression', + mapping: { + total_fields: { + limit: '10000', + }, + }, + // This is the default from Beats? So far seems to be a good value + refresh_interval: '5s', + // Default in the stack now, still good to have it in + number_of_shards: '1', + // We are setting 30 because it can be devided by several numbers. Useful when shrinking. + number_of_routing_shards: '30', + + // All the default fields which should be queried have to be added here. + // So far we add all keyword and text fields here if there are any, otherwise + // this setting is skipped. + ...(defaultFieldNames.length + ? { + query: { + default_field: defaultFieldNames, + }, + } + : {}), + }, + }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index e8dac60ddba1a..9dae415838890 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { merge } from 'lodash'; import Boom from '@hapi/boom'; import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; @@ -14,6 +15,7 @@ import type { IndexTemplateEntry, RegistryElasticsearch, InstallablePackage, + IndexTemplate, } from '../../../../types'; import { loadFieldsFromYaml, processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; @@ -32,6 +34,7 @@ import { getTemplate, getTemplatePriority, } from './template'; +import { buildDefaultSettings } from './default_settings'; export const installTemplates = async ( installablePackage: InstallablePackage, @@ -210,8 +213,9 @@ function buildComponentTemplates(params: { templateName: string; registryElasticsearch: RegistryElasticsearch | undefined; packageName: string; + defaultSettings: IndexTemplate['template']['settings']; }) { - const { templateName, registryElasticsearch, packageName } = params; + const { templateName, registryElasticsearch, packageName, defaultSettings } = params; const mappingsTemplateName = `${templateName}${mappingsSuffix}`; const settingsTemplateName = `${templateName}${settingsSuffix}`; const userSettingsTemplateName = `${templateName}${userSettingsSuffix}`; @@ -228,14 +232,12 @@ function buildComponentTemplates(params: { }; } - if (registryElasticsearch && registryElasticsearch['index_template.settings']) { - templatesMap[settingsTemplateName] = { - template: { - settings: registryElasticsearch['index_template.settings'], - }, - _meta, - }; - } + templatesMap[settingsTemplateName] = { + template: { + settings: merge(defaultSettings, registryElasticsearch?.['index_template.settings'] ?? {}), + }, + _meta, + }; // return empty/stub template templatesMap[userSettingsTemplateName] = { @@ -253,9 +255,15 @@ async function installDataStreamComponentTemplates(params: { registryElasticsearch: RegistryElasticsearch | undefined; esClient: ElasticsearchClient; packageName: string; + defaultSettings: IndexTemplate['template']['settings']; }) { - const { templateName, registryElasticsearch, esClient, packageName } = params; - const templates = buildComponentTemplates({ templateName, registryElasticsearch, packageName }); + const { templateName, registryElasticsearch, esClient, packageName, defaultSettings } = params; + const templates = buildComponentTemplates({ + templateName, + registryElasticsearch, + packageName, + defaultSettings, + }); const templateNames = Object.keys(templates); const templateEntries = Object.entries(templates); @@ -362,11 +370,20 @@ export async function installTemplate({ await esClient.indices.putIndexTemplate(updateIndexTemplateParams, { ignore: [404] }); } + const defaultSettings = buildDefaultSettings({ + templateName, + packageName, + fields, + type: dataStream.type, + ilmPolicy: dataStream.ilm_policy, + }); + const composedOfTemplates = await installDataStreamComponentTemplates({ templateName, registryElasticsearch: dataStream.elasticsearch, esClient, packageName, + defaultSettings, }); const template = getTemplate({ @@ -378,7 +395,6 @@ export async function installTemplate({ packageName, composedOfTemplates, templatePriority, - ilmPolicy: dataStream.ilm_policy, hidden: dataStream.hidden, }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index c999a135e2116..44d633d5f6e53 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -40,9 +40,6 @@ const DEFAULT_IGNORE_ABOVE = 1024; const DEFAULT_TEMPLATE_PRIORITY = 200; const DATASET_IS_PREFIX_TEMPLATE_PRIORITY = 150; -const QUERY_DEFAULT_FIELD_TYPES = ['keyword', 'text']; -const QUERY_DEFAULT_FIELD_LIMIT = 1024; - const META_PROP_KEYS = ['metric_type', 'unit']; /** @@ -59,7 +56,6 @@ export function getTemplate({ packageName, composedOfTemplates, templatePriority, - ilmPolicy, hidden, }: { type: string; @@ -70,7 +66,6 @@ export function getTemplate({ packageName: string; composedOfTemplates: string[]; templatePriority: number; - ilmPolicy?: string | undefined; hidden?: boolean; }): IndexTemplate { const template = getBaseTemplate( @@ -81,7 +76,6 @@ export function getTemplate({ packageName, composedOfTemplates, templatePriority, - ilmPolicy, hidden ); if (pipelineName) { @@ -370,11 +364,8 @@ function getBaseTemplate( packageName: string, composedOfTemplates: string[], templatePriority: number, - ilmPolicy?: string | undefined, hidden?: boolean ): IndexTemplate { - const logger = appContextService.getLogger(); - // Meta information to identify Ingest Manager's managed templates and indices const _meta = { package: { @@ -384,57 +375,13 @@ function getBaseTemplate( managed: true, }; - // Find all field names to set `index.query.default_field` to, which will be - // the first 1024 keyword or text fields - const defaultFields = flattenFieldsToNameAndType(fields).filter( - (field) => field.type && QUERY_DEFAULT_FIELD_TYPES.includes(field.type) - ); - if (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT) { - logger.warn( - `large amount of default fields detected for index template ${templateIndexPattern} in package ${packageName}, applying the first ${QUERY_DEFAULT_FIELD_LIMIT} fields` - ); - } - const defaultFieldNames = (defaultFields.length > QUERY_DEFAULT_FIELD_LIMIT - ? defaultFields.slice(0, QUERY_DEFAULT_FIELD_LIMIT) - : defaultFields - ).map((field) => field.name); - return { priority: templatePriority, // To be completed with the correct index patterns index_patterns: [templateIndexPattern], template: { settings: { - index: { - // ILM Policy must be added here, for now point to the default global ILM policy name - lifecycle: { - name: ilmPolicy ? ilmPolicy : type, - }, - // What should be our default for the compression? - codec: 'best_compression', - // W - mapping: { - total_fields: { - limit: '10000', - }, - }, - // This is the default from Beats? So far seems to be a good value - refresh_interval: '5s', - // Default in the stack now, still good to have it in - number_of_shards: '1', - // We are setting 30 because it can be devided by several numbers. Useful when shrinking. - number_of_routing_shards: '30', - // All the default fields which should be queried have to be added here. - // So far we add all keyword and text fields here if there are any, otherwise - // this setting is skipped. - ...(defaultFieldNames.length - ? { - query: { - default_field: defaultFieldNames, - }, - } - : {}), - }, + index: {}, }, mappings: { // All the dynamic field mappings diff --git a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap index 13c2dd24f9103..8f9428d8a12db 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap +++ b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap @@ -341,6 +341,10 @@ Object { "id": "logs-apache.access", "type": "index_template", }, + Object { + "id": "logs-apache.access@settings", + "type": "component_template", + }, Object { "id": "logs-apache.access@custom", "type": "component_template", @@ -349,6 +353,10 @@ Object { "id": "metrics-apache.status", "type": "index_template", }, + Object { + "id": "metrics-apache.status@settings", + "type": "component_template", + }, Object { "id": "metrics-apache.status@custom", "type": "component_template", @@ -357,6 +365,10 @@ Object { "id": "logs-apache.error", "type": "index_template", }, + Object { + "id": "logs-apache.error@settings", + "type": "component_template", + }, Object { "id": "logs-apache.error@custom", "type": "component_template", diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts index 06130775ec3cb..be1007e632ef4 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_by_upload.ts @@ -70,7 +70,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/gzip') .send(buf) .expect(200); - expect(res.body.response.length).to.be(26); + expect(res.body.response.length).to.be(29); }); it('should install a zip archive correctly and package info should return correctly after validation', async function () { @@ -81,7 +81,7 @@ export default function (providerContext: FtrProviderContext) { .type('application/zip') .send(buf) .expect(200); - expect(res.body.response.length).to.be(26); + expect(res.body.response.length).to.be(29); const packageInfoRes = await supertest .get(`/api/fleet/epm/packages/${testPkgKey}`) diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts index 770502db49dae..d1c3eae785f47 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts @@ -115,10 +115,18 @@ export default function (providerContext: FtrProviderContext) { template: { settings: { index: { + codec: 'best_compression', lifecycle: { name: 'overridden by user', }, + mapping: { + total_fields: { + limit: '10000', + }, + }, + number_of_routing_shards: 30, number_of_shards: '3', + refresh_interval: '5s', }, }, mappings: { diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 85573560177ee..02ecc9570afef 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -538,6 +538,10 @@ const expectAssetsInstalled = ({ id: 'logs-all_assets.test_logs@custom', type: 'component_template', }, + { + id: 'metrics-all_assets.test_metrics@settings', + type: 'component_template', + }, { id: 'metrics-all_assets.test_metrics@custom', type: 'component_template', diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 6b4d104423144..8c59533ce98dc 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -214,7 +214,21 @@ export default function (providerContext: FtrProviderContext) { }); expect(resSettings.statusCode).equal(200); expect(resSettings.body.component_templates[0].component_template.template.settings).eql({ - index: { lifecycle: { name: 'reference2' } }, + index: { + lifecycle: { name: 'reference2' }, + codec: 'best_compression', + mapping: { + total_fields: { + limit: '10000', + }, + }, + number_of_routing_shards: '30', + number_of_shards: '1', + query: { + default_field: ['logs_test_name', 'new_field_name'], + }, + refresh_interval: '5s', + }, }); const resUserSettings = await es.transport.request({ method: 'GET', @@ -359,6 +373,10 @@ export default function (providerContext: FtrProviderContext) { id: 'logs-all_assets.test_logs2', type: 'index_template', }, + { + id: 'logs-all_assets.test_logs2@settings', + type: 'component_template', + }, { id: 'logs-all_assets.test_logs2@custom', type: 'component_template', @@ -367,6 +385,10 @@ export default function (providerContext: FtrProviderContext) { id: 'metrics-all_assets.test_metrics', type: 'index_template', }, + { + id: 'metrics-all_assets.test_metrics@settings', + type: 'component_template', + }, { id: 'metrics-all_assets.test_metrics@custom', type: 'component_template',