diff --git a/src/core/server/config/deprecation/core_deprecations.ts b/src/core/server/config/deprecation/core_deprecations.ts index c63c9384da9d8..3aa7f9e2aa8ad 100644 --- a/src/core/server/config/deprecation/core_deprecations.ts +++ b/src/core/server/config/deprecation/core_deprecations.ts @@ -119,6 +119,56 @@ export const coreDeprecationProvider: ConfigDeprecationProvider = ({ renameFromRoot('xpack.telemetry.config', 'telemetry.config'), renameFromRoot('xpack.telemetry.banner', 'telemetry.banner'), renameFromRoot('xpack.telemetry.url', 'telemetry.url'), + // Monitoring renames + // TODO: Remove these from here once the monitoring plugin is migrated to NP + renameFromRoot('xpack.monitoring.enabled', 'monitoring.enabled'), + renameFromRoot('xpack.monitoring.ui.enabled', 'monitoring.ui.enabled'), + renameFromRoot( + 'xpack.monitoring.kibana.collection.enabled', + 'monitoring.kibana.collection.enabled' + ), + renameFromRoot('xpack.monitoring.max_bucket_size', 'monitoring.ui.max_bucket_size'), + renameFromRoot('xpack.monitoring.min_interval_seconds', 'monitoring.ui.min_interval_seconds'), + renameFromRoot( + 'xpack.monitoring.show_license_expiration', + 'monitoring.ui.show_license_expiration' + ), + renameFromRoot( + 'xpack.monitoring.ui.container.elasticsearch.enabled', + 'monitoring.ui.container.elasticsearch.enabled' + ), + renameFromRoot( + 'xpack.monitoring.ui.container.logstash.enabled', + 'monitoring.ui.container.logstash.enabled' + ), + renameFromRoot( + 'xpack.monitoring.tests.cloud_detector.enabled', + 'monitoring.tests.cloud_detector.enabled' + ), + renameFromRoot( + 'xpack.monitoring.kibana.collection.interval', + 'monitoring.kibana.collection.interval' + ), + renameFromRoot('xpack.monitoring.elasticsearch.hosts', 'monitoring.ui.elasticsearch.hosts'), + renameFromRoot('xpack.monitoring.elasticsearch.username', 'monitoring.ui.elasticsearch.username'), + renameFromRoot('xpack.monitoring.elasticsearch.password', 'monitoring.ui.elasticsearch.password'), + renameFromRoot( + 'xpack.monitoring.xpack_api_polling_frequency_millis', + 'monitoring.xpack_api_polling_frequency_millis' + ), + renameFromRoot( + 'xpack.monitoring.cluster_alerts.email_notifications.enabled', + 'monitoring.cluster_alerts.email_notifications.enabled' + ), + renameFromRoot( + 'xpack.monitoring.cluster_alerts.email_notifications.email_address', + 'monitoring.cluster_alerts.email_notifications.email_address' + ), + renameFromRoot('xpack.monitoring.ccs.enabled', 'monitoring.ui.ccs.enabled'), + renameFromRoot( + 'xpack.monitoring.elasticsearch.logFetchCount', + 'monitoring.ui.elasticsearch.logFetchCount' + ), configPathDeprecation, dataPathDeprecation, rewriteBasePathDeprecation, diff --git a/src/core/server/legacy/logging/appenders/legacy_appender.ts b/src/core/server/legacy/logging/appenders/legacy_appender.ts index 6d82d929e7daa..0c2f4ce93c3b8 100644 --- a/src/core/server/legacy/logging/appenders/legacy_appender.ts +++ b/src/core/server/legacy/logging/appenders/legacy_appender.ts @@ -33,6 +33,12 @@ export class LegacyAppender implements DisposableAppender { legacyLoggingConfig: schema.any(), }); + /** + * Sets {@link Appender.receiveAllLevels} because legacy does its own filtering based on the legacy logging + * configuration. + */ + public readonly receiveAllLevels = true; + private readonly loggingServer: LegacyLoggingServer; constructor(legacyLoggingConfig: Readonly) { diff --git a/src/core/server/logging/appenders/appenders.ts b/src/core/server/logging/appenders/appenders.ts index 3aa86495e4d82..871acb8c465ca 100644 --- a/src/core/server/logging/appenders/appenders.ts +++ b/src/core/server/logging/appenders/appenders.ts @@ -42,6 +42,12 @@ export type AppenderConfigType = TypeOf; */ export interface Appender { append(record: LogRecord): void; + + /** + * Used to signal to `Logger` that log level filtering should be ignored for this appender. Defaults to `false`. + * @deprecated Should be removed once the `LegacyAppender` is removed. + */ + receiveAllLevels?: boolean; } /** diff --git a/src/core/server/logging/logger.test.ts b/src/core/server/logging/logger.test.ts index 026e24fc5df54..eeebb8ad5a0fa 100644 --- a/src/core/server/logging/logger.test.ts +++ b/src/core/server/logging/logger.test.ts @@ -410,3 +410,85 @@ test('passes log record to appenders only if log level is supported.', () => { }); } }); + +test('passes log record to appender with receiveAllLevels: true, regardless if log level is supported', () => { + const receiveAllAppender = { append: jest.fn(), receiveAllLevels: true }; + const warnLogger = new BaseLogger(context, LogLevel.Warn, [receiveAllAppender], factory); + + warnLogger.trace('trace-message'); + expect(receiveAllAppender.append).toHaveBeenCalledTimes(1); + expect(receiveAllAppender.append.mock.calls[0][0]).toMatchObject({ + level: LogLevel.Trace, + message: 'trace-message', + }); + + warnLogger.debug('debug-message'); + expect(receiveAllAppender.append).toHaveBeenCalledTimes(2); + expect(receiveAllAppender.append.mock.calls[1][0]).toMatchObject({ + level: LogLevel.Debug, + message: 'debug-message', + }); + + warnLogger.info('info-message'); + expect(receiveAllAppender.append).toHaveBeenCalledTimes(3); + expect(receiveAllAppender.append.mock.calls[2][0]).toMatchObject({ + level: LogLevel.Info, + message: 'info-message', + }); + + warnLogger.warn('warn-message'); + expect(receiveAllAppender.append).toHaveBeenCalledTimes(4); + expect(receiveAllAppender.append.mock.calls[3][0]).toMatchObject({ + level: LogLevel.Warn, + message: 'warn-message', + }); + + warnLogger.error('error-message'); + expect(receiveAllAppender.append).toHaveBeenCalledTimes(5); + expect(receiveAllAppender.append.mock.calls[4][0]).toMatchObject({ + level: LogLevel.Error, + message: 'error-message', + }); + + warnLogger.fatal('fatal-message'); + expect(receiveAllAppender.append).toHaveBeenCalledTimes(6); + expect(receiveAllAppender.append.mock.calls[5][0]).toMatchObject({ + level: LogLevel.Fatal, + message: 'fatal-message', + }); +}); + +test('passes log record to appender with receiveAllLevels: false, only if log level is supported', () => { + const notReceiveAllAppender = { append: jest.fn(), receiveAllLevels: false }; + const warnLogger = new BaseLogger(context, LogLevel.Warn, [notReceiveAllAppender], factory); + + warnLogger.trace('trace-message'); + expect(notReceiveAllAppender.append).toHaveBeenCalledTimes(0); + + warnLogger.debug('debug-message'); + expect(notReceiveAllAppender.append).toHaveBeenCalledTimes(0); + + warnLogger.info('info-message'); + expect(notReceiveAllAppender.append).toHaveBeenCalledTimes(0); + + warnLogger.warn('warn-message'); + expect(notReceiveAllAppender.append).toHaveBeenCalledTimes(1); + expect(notReceiveAllAppender.append.mock.calls[0][0]).toMatchObject({ + level: LogLevel.Warn, + message: 'warn-message', + }); + + warnLogger.error('error-message'); + expect(notReceiveAllAppender.append).toHaveBeenCalledTimes(2); + expect(notReceiveAllAppender.append.mock.calls[1][0]).toMatchObject({ + level: LogLevel.Error, + message: 'error-message', + }); + + warnLogger.fatal('fatal-message'); + expect(notReceiveAllAppender.append).toHaveBeenCalledTimes(3); + expect(notReceiveAllAppender.append.mock.calls[2][0]).toMatchObject({ + level: LogLevel.Fatal, + message: 'fatal-message', + }); +}); diff --git a/src/core/server/logging/logger.ts b/src/core/server/logging/logger.ts index ac79c1916c07b..ab6906ff5d684 100644 --- a/src/core/server/logging/logger.ts +++ b/src/core/server/logging/logger.ts @@ -136,12 +136,12 @@ export class BaseLogger implements Logger { } public log(record: LogRecord) { - if (!this.level.supports(record.level)) { - return; - } + const supportedLevel = this.level.supports(record.level); for (const appender of this.appenders) { - appender.append(record); + if (supportedLevel || appender.receiveAllLevels) { + appender.append(record); + } } } diff --git a/x-pack/legacy/plugins/monitoring/config.js b/x-pack/legacy/plugins/monitoring/config.js index 91c1ee99a0b2e..778b656c056f2 100644 --- a/x-pack/legacy/plugins/monitoring/config.js +++ b/x-pack/legacy/plugins/monitoring/config.js @@ -15,12 +15,12 @@ export const config = Joi => { const DEFAULT_REQUEST_HEADERS = ['authorization']; return Joi.object({ - ccs: Joi.object({ - enabled: Joi.boolean().default(true), - }).default(), enabled: Joi.boolean().default(true), ui: Joi.object({ enabled: Joi.boolean().default(true), + ccs: Joi.object({ + enabled: Joi.boolean().default(true), + }).default(), container: Joi.object({ elasticsearch: Joi.object({ enabled: Joi.boolean().default(false), @@ -29,6 +29,51 @@ export const config = Joi => { enabled: Joi.boolean().default(false), }).default(), }).default(), + max_bucket_size: Joi.number().default(10000), + min_interval_seconds: Joi.number().default(10), + show_license_expiration: Joi.boolean().default(true), + elasticsearch: Joi.object({ + customHeaders: Joi.object().default({}), + logQueries: Joi.boolean().default(false), + requestHeadersWhitelist: Joi.array() + .items() + .single() + .default(DEFAULT_REQUEST_HEADERS), + sniffOnStart: Joi.boolean().default(false), + sniffInterval: Joi.number() + .allow(false) + .default(false), + sniffOnConnectionFault: Joi.boolean().default(false), + hosts: Joi.array() + .items(Joi.string().uri({ scheme: ['http', 'https'] })) + .single(), // if empty, use Kibana's connection config + username: Joi.string(), + password: Joi.string(), + requestTimeout: Joi.number().default(30000), + pingTimeout: Joi.number().default(30000), + ssl: Joi.object({ + verificationMode: Joi.string() + .valid('none', 'certificate', 'full') + .default('full'), + certificateAuthorities: Joi.array() + .single() + .items(Joi.string()), + certificate: Joi.string(), + key: Joi.string(), + keyPassphrase: Joi.string(), + keystore: Joi.object({ + path: Joi.string(), + password: Joi.string(), + }).default(), + truststore: Joi.object({ + path: Joi.string(), + password: Joi.string(), + }).default(), + alwaysPresentCertificate: Joi.boolean().default(false), + }).default(), + apiVersion: Joi.string().default('master'), + logFetchCount: Joi.number().default(10), + }).default(), }).default(), kibana: Joi.object({ collection: Joi.object({ @@ -46,56 +91,11 @@ export const config = Joi => { xpack_api_polling_frequency_millis: Joi.number().default( XPACK_INFO_API_DEFAULT_POLL_FREQUENCY_IN_MILLIS ), - max_bucket_size: Joi.number().default(10000), - min_interval_seconds: Joi.number().default(10), - show_license_expiration: Joi.boolean().default(true), agent: Joi.object({ interval: Joi.string() .regex(/[\d\.]+[yMwdhms]/) .default('10s'), }).default(), - elasticsearch: Joi.object({ - customHeaders: Joi.object().default({}), - logQueries: Joi.boolean().default(false), - requestHeadersWhitelist: Joi.array() - .items() - .single() - .default(DEFAULT_REQUEST_HEADERS), - sniffOnStart: Joi.boolean().default(false), - sniffInterval: Joi.number() - .allow(false) - .default(false), - sniffOnConnectionFault: Joi.boolean().default(false), - hosts: Joi.array() - .items(Joi.string().uri({ scheme: ['http', 'https'] })) - .single(), // if empty, use Kibana's connection config - username: Joi.string(), - password: Joi.string(), - requestTimeout: Joi.number().default(30000), - pingTimeout: Joi.number().default(30000), - ssl: Joi.object({ - verificationMode: Joi.string() - .valid('none', 'certificate', 'full') - .default('full'), - certificateAuthorities: Joi.array() - .single() - .items(Joi.string()), - certificate: Joi.string(), - key: Joi.string(), - keyPassphrase: Joi.string(), - keystore: Joi.object({ - path: Joi.string(), - password: Joi.string(), - }).default(), - truststore: Joi.object({ - path: Joi.string(), - password: Joi.string(), - }).default(), - alwaysPresentCertificate: Joi.boolean().default(false), - }).default(), - apiVersion: Joi.string().default('master'), - logFetchCount: Joi.number().default(10), - }).default(), tests: Joi.object({ cloud_detector: Joi.object({ enabled: Joi.boolean().default(true), diff --git a/x-pack/legacy/plugins/monitoring/index.js b/x-pack/legacy/plugins/monitoring/index.js index 37003ca058a63..8145d89b2db31 100644 --- a/x-pack/legacy/plugins/monitoring/index.js +++ b/x-pack/legacy/plugins/monitoring/index.js @@ -20,32 +20,32 @@ export const monitoring = kibana => new kibana.Plugin({ require: ['kibana', 'elasticsearch', 'xpack_main'], id: 'monitoring', - configPrefix: 'xpack.monitoring', + configPrefix: 'monitoring', publicDir: resolve(__dirname, 'public'), init(server) { const configs = [ - 'xpack.monitoring.ui.enabled', - 'xpack.monitoring.kibana.collection.enabled', - 'xpack.monitoring.max_bucket_size', - 'xpack.monitoring.min_interval_seconds', + 'monitoring.ui.enabled', + 'monitoring.kibana.collection.enabled', + 'monitoring.ui.max_bucket_size', + 'monitoring.ui.min_interval_seconds', 'kibana.index', 'pkg.version', - 'xpack.monitoring.show_license_expiration', - 'xpack.monitoring.ui.container.elasticsearch.enabled', - 'xpack.monitoring.ui.container.logstash.enabled', - 'xpack.monitoring.tests.cloud_detector.enabled', - 'xpack.monitoring.kibana.collection.interval', - 'xpack.monitoring.elasticsearch.hosts', - 'xpack.monitoring.elasticsearch', - 'xpack.monitoring.xpack_api_polling_frequency_millis', + 'monitoring.ui.show_license_expiration', + 'monitoring.ui.container.elasticsearch.enabled', + 'monitoring.ui.container.logstash.enabled', + 'monitoring.tests.cloud_detector.enabled', + 'monitoring.kibana.collection.interval', + 'monitoring.ui.elasticsearch.hosts', + 'monitoring.ui.elasticsearch', + 'monitoring.xpack_api_polling_frequency_millis', 'server.uuid', 'server.name', 'server.host', 'server.port', - 'xpack.monitoring.cluster_alerts.email_notifications.enabled', - 'xpack.monitoring.cluster_alerts.email_notifications.email_address', - 'xpack.monitoring.ccs.enabled', - 'xpack.monitoring.elasticsearch.logFetchCount', + 'monitoring.cluster_alerts.email_notifications.enabled', + 'monitoring.cluster_alerts.email_notifications.email_address', + 'monitoring.ui.ccs.enabled', + 'monitoring.ui.elasticsearch.logFetchCount', ]; const serverConfig = server.config(); diff --git a/x-pack/legacy/plugins/monitoring/server/cluster_alerts/alerts_cluster_search.js b/x-pack/legacy/plugins/monitoring/server/cluster_alerts/alerts_cluster_search.js index 0c9fb4bd04ee7..eff9875d794ad 100644 --- a/x-pack/legacy/plugins/monitoring/server/cluster_alerts/alerts_cluster_search.js +++ b/x-pack/legacy/plugins/monitoring/server/cluster_alerts/alerts_cluster_search.js @@ -157,7 +157,7 @@ export function alertsClusterSearch(req, alertsIndex, cluster, checkLicense, opt if (prodLicenseInfo.clusterAlerts.enabled) { const config = req.server.config(); - const size = options.size || config.get('xpack.monitoring.max_bucket_size'); + const size = options.size || config.get('monitoring.ui.max_bucket_size'); const params = { index: alertsIndex, diff --git a/x-pack/legacy/plugins/monitoring/server/cluster_alerts/verify_monitoring_license.js b/x-pack/legacy/plugins/monitoring/server/cluster_alerts/verify_monitoring_license.js index 9cc67e11c28d5..e94f4e08fbdb1 100644 --- a/x-pack/legacy/plugins/monitoring/server/cluster_alerts/verify_monitoring_license.js +++ b/x-pack/legacy/plugins/monitoring/server/cluster_alerts/verify_monitoring_license.js @@ -19,7 +19,7 @@ export function verifyMonitoringLicense(server) { const config = server.config(); // if cluster alerts are enabled, then ensure that we can use it according to the license - if (config.get('xpack.monitoring.cluster_alerts.enabled')) { + if (config.get('monitoring.cluster_alerts.enabled')) { const xpackInfo = get(server.plugins.monitoring, 'info'); if (xpackInfo) { const monitoringCluster = xpackInfo.feature('monitoring').getLicenseCheckResults(); diff --git a/x-pack/legacy/plugins/monitoring/server/es_client/__tests__/instantiate_client.js b/x-pack/legacy/plugins/monitoring/server/es_client/__tests__/instantiate_client.js index 6844bd5febf8e..88cf9734d5f57 100644 --- a/x-pack/legacy/plugins/monitoring/server/es_client/__tests__/instantiate_client.js +++ b/x-pack/legacy/plugins/monitoring/server/es_client/__tests__/instantiate_client.js @@ -11,8 +11,8 @@ import { exposeClient, hasMonitoringCluster } from '../instantiate_client'; function getMockServerFromConnectionUrl(monitoringClusterUrl) { const server = { - xpack: { - monitoring: { + monitoring: { + ui: { elasticsearch: { hosts: monitoringClusterUrl ? [monitoringClusterUrl] : [], username: 'monitoring-user-internal-test', @@ -27,7 +27,7 @@ function getMockServerFromConnectionUrl(monitoringClusterUrl) { }; return { - elasticsearchConfig: server.xpack.monitoring.elasticsearch, + elasticsearchConfig: server.monitoring.ui.elasticsearch, elasticsearchPlugin: { getCluster: sinon .stub() diff --git a/x-pack/legacy/plugins/monitoring/server/es_client/parse_elasticsearch_config.test.ts b/x-pack/legacy/plugins/monitoring/server/es_client/parse_elasticsearch_config.test.ts index c6f4e0fa68504..8d9b5335732c0 100644 --- a/x-pack/legacy/plugins/monitoring/server/es_client/parse_elasticsearch_config.test.ts +++ b/x-pack/legacy/plugins/monitoring/server/es_client/parse_elasticsearch_config.test.ts @@ -168,14 +168,14 @@ describe('throws when config is invalid', () => { it('throws if key and keystore.path are both specified', () => { const value = { ssl: { key: 'foo', keystore: { path: 'bar' } } }; expect(() => parse(value)).toThrowErrorMatchingInlineSnapshot( - `"[config validation of [xpack.monitoring.elasticsearch].ssl]: cannot use [key] when [keystore.path] is specified"` + `"[config validation of [monitoring.ui.elasticsearch].ssl]: cannot use [key] when [keystore.path] is specified"` ); }); it('throws if certificate and keystore.path are both specified', () => { const value = { ssl: { certificate: 'foo', keystore: { path: 'bar' } } }; expect(() => parse(value)).toThrowErrorMatchingInlineSnapshot( - `"[config validation of [xpack.monitoring.elasticsearch].ssl]: cannot use [certificate] when [keystore.path] is specified"` + `"[config validation of [monitoring.ui.elasticsearch].ssl]: cannot use [certificate] when [keystore.path] is specified"` ); }); }); diff --git a/x-pack/legacy/plugins/monitoring/server/es_client/parse_elasticsearch_config.ts b/x-pack/legacy/plugins/monitoring/server/es_client/parse_elasticsearch_config.ts index 70e6235602b5b..728b3433bf06c 100644 --- a/x-pack/legacy/plugins/monitoring/server/es_client/parse_elasticsearch_config.ts +++ b/x-pack/legacy/plugins/monitoring/server/es_client/parse_elasticsearch_config.ts @@ -7,7 +7,7 @@ import { readFileSync } from 'fs'; import { readPkcs12Truststore, readPkcs12Keystore } from '../../../../../../src/core/utils'; -const KEY = 'xpack.monitoring.elasticsearch'; +const KEY = 'monitoring.ui.elasticsearch'; /* * Parse a config object's Elasticsearch configuration, reading any diff --git a/x-pack/legacy/plugins/monitoring/server/init_monitoring_xpack_info.js b/x-pack/legacy/plugins/monitoring/server/init_monitoring_xpack_info.js index b43430ead23b0..ba07f512de896 100644 --- a/x-pack/legacy/plugins/monitoring/server/init_monitoring_xpack_info.js +++ b/x-pack/legacy/plugins/monitoring/server/init_monitoring_xpack_info.js @@ -15,7 +15,7 @@ export const initMonitoringXpackInfo = async ({ config, xpackMainPlugin, expose, const xpackInfo = hasMonitoringCluster(config) ? xpackMainPlugin.createXPackInfo({ clusterSource: 'monitoring', - pollFrequencyInMillis: config.get('xpack.monitoring.xpack_api_polling_frequency_millis'), + pollFrequencyInMillis: config.get('monitoring.xpack_api_polling_frequency_millis'), }) : xpackMainPlugin.info; diff --git a/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/__tests__/get_default_admin_email.js b/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/__tests__/get_default_admin_email.js index a88f69af2aefb..59fb2952196ab 100644 --- a/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/__tests__/get_default_admin_email.js +++ b/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/__tests__/get_default_admin_email.js @@ -21,14 +21,10 @@ describe('getSettingsCollector / getDefaultAdminEmail', () => { }) { const config = { get: sinon.stub() }; - config.get - .withArgs('xpack.monitoring.cluster_alerts.email_notifications.enabled') - .returns(enabled); + config.get.withArgs('monitoring.cluster_alerts.email_notifications.enabled').returns(enabled); if (adminEmail) { - config.get - .withArgs(`xpack.monitoring.${CLUSTER_ALERTS_ADDRESS_CONFIG_KEY}`) - .returns(adminEmail); + config.get.withArgs(`monitoring.${CLUSTER_ALERTS_ADDRESS_CONFIG_KEY}`).returns(adminEmail); } config.get.withArgs('kibana.index').returns('.kibana'); @@ -74,7 +70,7 @@ describe('getSettingsCollector / getDefaultAdminEmail', () => { resetDeprecationWarning(); }); - describe('xpack.monitoring.cluster_alerts.email_notifications.enabled = false', () => { + describe('monitoring.cluster_alerts.email_notifications.enabled = false', () => { it('returns null', async () => { const { config, callCluster, log } = setup({ enabled: false }); expect(await getDefaultAdminEmail(config, callCluster, log)).to.be(null); @@ -131,12 +127,12 @@ describe('getSettingsCollector / getDefaultAdminEmail', () => { }); }); - describe('using xpack.monitoring.cluster_alerts.email_notifications.email_address', () => { + describe('using monitoring.cluster_alerts.email_notifications.email_address', () => { beforeEach(() => { resetDeprecationWarning(); }); - describe('xpack.monitoring.cluster_alerts.email_notifications.enabled = false', () => { + describe('monitoring.cluster_alerts.email_notifications.enabled = false', () => { it('returns null', async () => { const { config, callCluster, log } = setup({ enabled: false }); expect(await getDefaultAdminEmail(config, callCluster, log)).to.be(null); diff --git a/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js b/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js index 361de95653349..d04a4a1f669d9 100644 --- a/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js +++ b/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js @@ -19,11 +19,11 @@ export function resetDeprecationWarning() { * If so, use uiSettings API to fetch the X-Pack default admin email */ export async function getDefaultAdminEmail(config, callCluster, log) { - if (!config.get('xpack.monitoring.cluster_alerts.email_notifications.enabled')) { + if (!config.get('monitoring.cluster_alerts.email_notifications.enabled')) { return null; } - const emailAddressConfigKey = `xpack.monitoring.${CLUSTER_ALERTS_ADDRESS_CONFIG_KEY}`; + const emailAddressConfigKey = `monitoring.${CLUSTER_ALERTS_ADDRESS_CONFIG_KEY}`; const configuredEmailAddress = config.get(emailAddressConfigKey); if (configuredEmailAddress) { diff --git a/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js b/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js index d58f6f3254c76..699a364433b3e 100644 --- a/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js +++ b/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/collectors/ops_buffer/ops_buffer.js @@ -17,7 +17,7 @@ export function opsBuffer({ config, log, getOSInfo }) { // determine the cloud service in the background const cloudDetector = new CloudDetector(); - if (config.get('xpack.monitoring.tests.cloud_detector.enabled')) { + if (config.get('monitoring.tests.cloud_detector.enabled')) { cloudDetector.detectCloudService(); } diff --git a/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/init.js b/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/init.js index bf79ddc210902..3c02e2be58dec 100644 --- a/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/init.js +++ b/x-pack/legacy/plugins/monitoring/server/kibana_monitoring/init.js @@ -16,7 +16,7 @@ import { BulkUploader } from './bulk_uploader'; * @param {Object} server HapiJS server instance */ export function initBulkUploader({ config, ...params }) { - const interval = config.get('xpack.monitoring.kibana.collection.interval'); + const interval = config.get('monitoring.kibana.collection.interval'); return new BulkUploader({ interval, config, diff --git a/x-pack/legacy/plugins/monitoring/server/lib/__tests__/ccs_utils.js b/x-pack/legacy/plugins/monitoring/server/lib/__tests__/ccs_utils.js index 844dfc96bb19b..2d310962238fd 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/__tests__/ccs_utils.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/__tests__/ccs_utils.js @@ -17,7 +17,7 @@ describe('ccs_utils', () => { const get = sinon.stub(); const config = { get }; - get.withArgs('xpack.monitoring.ccs.enabled').returns(false); + get.withArgs('monitoring.ui.ccs.enabled').returns(false); // falsy string values should be ignored const allPattern = prefixIndexPattern(config, indexPattern, '*'); @@ -32,7 +32,7 @@ describe('ccs_utils', () => { const get = sinon.stub(); const config = { get }; - get.withArgs('xpack.monitoring.ccs.enabled').returns(true); + get.withArgs('monitoring.ui.ccs.enabled').returns(true); // falsy string values should be ignored const undefinedPattern = prefixIndexPattern(config, indexPattern); @@ -49,7 +49,7 @@ describe('ccs_utils', () => { const get = sinon.stub(); const config = { get }; - get.withArgs('xpack.monitoring.ccs.enabled').returns(true); + get.withArgs('monitoring.ui.ccs.enabled').returns(true); const abcPattern = prefixIndexPattern(config, indexPattern, 'aBc'); const underscorePattern = prefixIndexPattern(config, indexPattern, 'cluster_one'); @@ -67,7 +67,7 @@ describe('ccs_utils', () => { const get = sinon.stub(); const config = { get }; - get.withArgs('xpack.monitoring.ccs.enabled').returns(true); + get.withArgs('monitoring.ui.ccs.enabled').returns(true); const pattern = prefixIndexPattern(config, indexPattern, '*'); diff --git a/x-pack/legacy/plugins/monitoring/server/lib/apm/get_apms.js b/x-pack/legacy/plugins/monitoring/server/lib/apm/get_apms.js index ef8db59620f1a..40070a6b0d0f2 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/apm/get_apms.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/apm/get_apms.js @@ -84,7 +84,7 @@ export async function getApms(req, apmIndexPattern, clusterUuid) { const params = { index: apmIndexPattern, - size: config.get('xpack.monitoring.max_bucket_size'), // FIXME + size: config.get('monitoring.ui.max_bucket_size'), // FIXME ignoreUnavailable: true, filterPath: [ // only filter path can filter for inner_hits diff --git a/x-pack/legacy/plugins/monitoring/server/lib/apm/get_apms_for_clusters.js b/x-pack/legacy/plugins/monitoring/server/lib/apm/get_apms_for_clusters.js index 95ccb81f696be..a24936dc0f832 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/apm/get_apms_for_clusters.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/apm/get_apms_for_clusters.js @@ -35,7 +35,7 @@ export function getApmsForClusters(req, apmIndexPattern, clusters) { const start = req.payload.timeRange.min; const end = req.payload.timeRange.max; const config = req.server.config(); - const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); + const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); return Promise.all( clusters.map(async cluster => { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/apm/get_stats.js b/x-pack/legacy/plugins/monitoring/server/lib/apm/get_stats.js index 54a0609d945de..bfaec4f8a1294 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/apm/get_stats.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/apm/get_stats.js @@ -28,7 +28,7 @@ export async function getStats(req, apmIndexPattern, clusterUuid) { const config = req.server.config(); const start = moment.utc(req.payload.timeRange.min).valueOf(); const end = moment.utc(req.payload.timeRange.max).valueOf(); - const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); + const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const params = { index: apmIndexPattern, diff --git a/x-pack/legacy/plugins/monitoring/server/lib/beats/get_beats.js b/x-pack/legacy/plugins/monitoring/server/lib/beats/get_beats.js index 5857ec32b2259..ef878e4892557 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/beats/get_beats.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/beats/get_beats.js @@ -83,7 +83,7 @@ export async function getBeats(req, beatsIndexPattern, clusterUuid) { const params = { index: beatsIndexPattern, - size: config.get('xpack.monitoring.max_bucket_size'), // FIXME + size: config.get('monitoring.ui.max_bucket_size'), // FIXME ignoreUnavailable: true, filterPath: [ // only filter path can filter for inner_hits diff --git a/x-pack/legacy/plugins/monitoring/server/lib/beats/get_beats_for_clusters.js b/x-pack/legacy/plugins/monitoring/server/lib/beats/get_beats_for_clusters.js index 82a738755931d..624abb894e508 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/beats/get_beats_for_clusters.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/beats/get_beats_for_clusters.js @@ -34,7 +34,7 @@ export function getBeatsForClusters(req, beatsIndexPattern, clusters) { const start = req.payload.timeRange.min; const end = req.payload.timeRange.max; const config = req.server.config(); - const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); + const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); return Promise.all( clusters.map(async cluster => { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/beats/get_latest_stats.js b/x-pack/legacy/plugins/monitoring/server/lib/beats/get_latest_stats.js index d326c84634e12..1139489728dbf 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/beats/get_latest_stats.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/beats/get_latest_stats.js @@ -71,7 +71,7 @@ export function getLatestStats(req, beatsIndexPattern, clusterUuid) { uuids: { terms: { field: 'beats_stats.beat.uuid', - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), }, }, }, diff --git a/x-pack/legacy/plugins/monitoring/server/lib/beats/get_stats.js b/x-pack/legacy/plugins/monitoring/server/lib/beats/get_stats.js index 80851a8498c26..0f90750a293fb 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/beats/get_stats.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/beats/get_stats.js @@ -28,7 +28,7 @@ export async function getStats(req, beatsIndexPattern, clusterUuid) { const config = req.server.config(); const start = moment.utc(req.payload.timeRange.min).valueOf(); const end = moment.utc(req.payload.timeRange.max).valueOf(); - const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); + const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const params = { index: beatsIndexPattern, diff --git a/x-pack/legacy/plugins/monitoring/server/lib/ccs_utils.js b/x-pack/legacy/plugins/monitoring/server/lib/ccs_utils.js index 5b3980d9619a8..3409462156a07 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/ccs_utils.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/ccs_utils.js @@ -16,7 +16,7 @@ * @return {String} The index pattern with the {@code cluster} prefix appropriately prepended. */ export function prefixIndexPattern(config, indexPattern, ccs) { - const ccsEnabled = config.get('xpack.monitoring.ccs.enabled'); + const ccsEnabled = config.get('monitoring.ui.ccs.enabled'); if (!ccsEnabled || !ccs) { return indexPattern; diff --git a/x-pack/legacy/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js b/x-pack/legacy/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js index 856ee6c7576c4..d3456eeb2fe4e 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js @@ -34,6 +34,7 @@ import { checkCcrEnabled } from '../elasticsearch/ccr'; import { getStandaloneClusterDefinition, hasStandaloneClusters } from '../standalone_clusters'; import { getLogTypes } from '../logs'; import { isInCodePath } from './is_in_code_path'; +import { getLogstashPipelineIds } from '../logstash/get_pipeline_ids'; /** * Get all clusters or the cluster associated with {@code clusterUuid} when it is defined. @@ -53,6 +54,8 @@ export async function getClustersFromRequest( filebeatIndexPattern, } = indexPatterns; + const config = req.server.config(); + const size = config.get('xpack.monitoring.max_bucket_size'); const isStandaloneCluster = clusterUuid === STANDALONE_CLUSTER_CLUSTER_UUID; let clusters = []; @@ -158,25 +161,27 @@ export async function getClustersFromRequest( }); // add logstash data - const logstashes = isInCodePath(codePaths, [CODE_PATH_LOGSTASH]) - ? await getLogstashForClusters(req, lsIndexPattern, clusters) - : []; - - const clusterPipelineNodesCount = isInCodePath(codePaths, [CODE_PATH_LOGSTASH]) - ? await getPipelines(req, lsIndexPattern, null, ['logstash_cluster_pipeline_nodes_count']) - : []; - - // add the logstash data to each cluster - logstashes.forEach(logstash => { - const clusterIndex = findIndex(clusters, { cluster_uuid: logstash.clusterUuid }); - - // withhold LS overview stats until pipeline metrics have at least one full bucket - if (logstash.clusterUuid === req.params.clusterUuid && clusterPipelineNodesCount.length === 0) { - logstash.stats = {}; - } - - set(clusters[clusterIndex], 'logstash', logstash.stats); - }); + if (isInCodePath(codePaths, [CODE_PATH_LOGSTASH])) { + const logstashes = await getLogstashForClusters(req, lsIndexPattern, clusters); + const pipelines = await getLogstashPipelineIds(req, lsIndexPattern, { clusterUuid }, size); + const clusterPipelineNodesCount = await getPipelines(req, lsIndexPattern, pipelines, [ + 'logstash_cluster_pipeline_nodes_count', + ]); + // add the logstash data to each cluster + logstashes.forEach(logstash => { + const clusterIndex = findIndex(clusters, { cluster_uuid: logstash.clusterUuid }); + + // withhold LS overview stats until pipeline metrics have at least one full bucket + if ( + logstash.clusterUuid === req.params.clusterUuid && + clusterPipelineNodesCount.length === 0 + ) { + logstash.stats = {}; + } + + set(clusters[clusterIndex], 'logstash', logstash.stats); + }); + } // add beats data const beatsByCluster = isInCodePath(codePaths, [CODE_PATH_BEATS]) @@ -199,7 +204,6 @@ export async function getClustersFromRequest( // check ccr configuration const isCcrEnabled = await checkCcrEnabled(req, esIndexPattern); - const config = req.server.config(); const kibanaUuid = config.get('server.uuid'); return getClustersSummary(req.server, clusters, kibanaUuid, isCcrEnabled); diff --git a/x-pack/legacy/plugins/monitoring/server/lib/cluster/get_clusters_stats.js b/x-pack/legacy/plugins/monitoring/server/lib/cluster/get_clusters_stats.js index c323cb381aaf2..54dc58a374c2c 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/cluster/get_clusters_stats.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/cluster/get_clusters_stats.js @@ -46,7 +46,7 @@ function fetchClusterStats(req, esIndexPattern, clusterUuid) { const metric = ElasticsearchMetric.getMetricFields(); const params = { index: esIndexPattern, - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), ignoreUnavailable: true, filterPath: [ 'hits.hits._index', diff --git a/x-pack/legacy/plugins/monitoring/server/lib/details/__test__/get_metrics.test.js b/x-pack/legacy/plugins/monitoring/server/lib/details/__test__/get_metrics.test.js index b7c387e74ec96..fbe6c8ec4cfa3 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/details/__test__/get_metrics.test.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/details/__test__/get_metrics.test.js @@ -20,7 +20,7 @@ function getMockReq(metricsBuckets = []) { get: sinon.stub(), }; - config.get.withArgs('xpack.monitoring.min_interval_seconds').returns(10); + config.get.withArgs('monitoring.ui.min_interval_seconds').returns(10); return { server: { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/details/get_metrics.js b/x-pack/legacy/plugins/monitoring/server/lib/details/get_metrics.js index 798a94abbe484..0c4736e91ea10 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/details/get_metrics.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/details/get_metrics.js @@ -28,7 +28,7 @@ export async function getMetrics( // TODO: Pass in req parameters as explicit function parameters let min = moment.utc(req.payload.timeRange.min).valueOf(); const max = moment.utc(req.payload.timeRange.max).valueOf(); - const minIntervalSeconds = config.get('xpack.monitoring.min_interval_seconds'); + const minIntervalSeconds = config.get('monitoring.ui.min_interval_seconds'); const bucketSize = calculateTimeseriesInterval(min, max, minIntervalSeconds); const timezone = await getTimezone(req); diff --git a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/get_ml_jobs.js b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/get_ml_jobs.js index 658ee96c1f084..8aef402f881e8 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/get_ml_jobs.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/get_ml_jobs.js @@ -23,7 +23,7 @@ export function getMlJobs(req, esIndexPattern) { checkParam(esIndexPattern, 'esIndexPattern in getMlJobs'); const config = req.server.config(); - const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); + const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const start = req.payload.timeRange.min; // no wrapping in moment :) const end = req.payload.timeRange.max; const clusterUuid = req.params.clusterUuid; diff --git a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.js b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.js index 6fe8ccfd89043..938a9b9d55e43 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.js @@ -97,7 +97,7 @@ export function getIndices(req, esIndexPattern, showSystemIndices = false, shard const params = { index: esIndexPattern, // TODO: composite aggregation - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), ignoreUnavailable: true, filterPath: [ // only filter path can filter for inner_hits diff --git a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_nodes.js b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_nodes.js index 7581a32590971..c248ad743e0ec 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_nodes.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_nodes.js @@ -44,7 +44,7 @@ export async function getNodes(req, esIndexPattern, pageOfNodes, clusterStats, n const min = start; const bucketSize = Math.max( - config.get('xpack.monitoring.min_interval_seconds'), + config.get('monitoring.ui.min_interval_seconds'), calculateAuto(100, duration).asSeconds() ); @@ -59,7 +59,7 @@ export async function getNodes(req, esIndexPattern, pageOfNodes, clusterStats, n const params = { index: esIndexPattern, - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), ignoreUnavailable: true, body: { query: createQuery({ @@ -78,7 +78,7 @@ export async function getNodes(req, esIndexPattern, pageOfNodes, clusterStats, n terms: { field: `source_node.uuid`, include: uuidsToInclude, - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), }, aggs: { by_date: { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.js b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.js index 51c61046e9cda..e18d328e8725b 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_paginated_nodes.js @@ -38,7 +38,7 @@ export async function getPaginatedNodes( { clusterStats, nodesShardCount } ) { const config = req.server.config(); - const size = config.get('xpack.monitoring.max_bucket_size'); + const size = config.get('monitoring.ui.max_bucket_size'); const nodes = await getNodeIds(req, esIndexPattern, { clusterUuid }, size); // Add `isOnline` and shards from the cluster state and shard stats @@ -63,7 +63,7 @@ export async function getPaginatedNodes( const groupBy = { field: `source_node.uuid`, include: nodes.map(node => node.uuid), - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), }; const metricSeriesData = await getMetrics( req, diff --git a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.js b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.js index e8d484e7021f4..c77bcc4f62e61 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.js @@ -12,7 +12,7 @@ import { calculateIndicesTotals } from './calculate_shard_stat_indices_totals'; async function getUnassignedShardData(req, esIndexPattern, cluster) { const config = req.server.config(); - const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); + const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const metric = ElasticsearchMetric.getMetricFields(); const params = { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.js b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.js index c11bd4aead693..7823884dc749d 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.js @@ -11,7 +11,7 @@ import { ElasticsearchMetric } from '../../metrics'; async function getShardCountPerNode(req, esIndexPattern, cluster) { const config = req.server.config(); - const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); + const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const metric = ElasticsearchMetric.getMetricFields(); const params = { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.js b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.js index 3be5650b7d3bc..40412c03b0ef9 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.js @@ -55,7 +55,7 @@ export function getShardAllocation( const metric = ElasticsearchMetric.getMetricFields(); const params = { index: esIndexPattern, - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), ignoreUnavailable: true, body: { query: createQuery({ type: 'shards', clusterUuid, metric, filters }), diff --git a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stat_aggs.js b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stat_aggs.js index eddd50612cdb1..8c4834e5d5e40 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stat_aggs.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stat_aggs.js @@ -9,7 +9,7 @@ * @param {Boolean} includeNodes - whether to add the aggs for node shards */ export function getShardAggs(config, includeNodes, includeIndices) { - const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); + const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const aggSize = 10; const indicesAgg = { terms: { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/kibana/get_kibanas.js b/x-pack/legacy/plugins/monitoring/server/lib/kibana/get_kibanas.js index af6563bae682d..c272c38f00d55 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/kibana/get_kibanas.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/kibana/get_kibanas.js @@ -31,7 +31,7 @@ export function getKibanas(req, kbnIndexPattern, { clusterUuid }) { const params = { index: kbnIndexPattern, - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), ignoreUnavailable: true, body: { query: createQuery({ diff --git a/x-pack/legacy/plugins/monitoring/server/lib/kibana/get_kibanas_for_clusters.js b/x-pack/legacy/plugins/monitoring/server/lib/kibana/get_kibanas_for_clusters.js index dbf1c41dcf4e5..e50e8bda3c907 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/kibana/get_kibanas_for_clusters.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/kibana/get_kibanas_for_clusters.js @@ -49,7 +49,7 @@ export function getKibanasForClusters(req, kbnIndexPattern, clusters) { kibana_uuids: { terms: { field: 'kibana_stats.kibana.uuid', - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), }, aggs: { latest_report: { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logs/get_logs.js b/x-pack/legacy/plugins/monitoring/server/lib/logs/get_logs.js index 7a20d7737c5e8..b876e3ba05d70 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logs/get_logs.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logs/get_logs.js @@ -70,7 +70,7 @@ export async function getLogs( const params = { index: filebeatIndexPattern, - size: Math.min(50, config.get('xpack.monitoring.elasticsearch.logFetchCount')), + size: Math.min(50, config.get('monitoring.ui.elasticsearch.logFetchCount')), filterPath: [ 'hits.hits._source.message', 'hits.hits._source.log.level', diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/__tests__/get_pipelines.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/__tests__/get_pipelines.js index dc9f48785bea7..cac77b2903439 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/__tests__/get_pipelines.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/__tests__/get_pipelines.js @@ -5,7 +5,7 @@ */ import expect from '@kbn/expect'; -import { handleGetPipelinesResponse, processPipelinesAPIResponse } from '../get_pipelines'; +import { processPipelinesAPIResponse } from '../get_pipelines'; describe('processPipelinesAPIResponse', () => { let response; @@ -13,6 +13,7 @@ describe('processPipelinesAPIResponse', () => { response = { pipelines: [ { + id: 1, metrics: { throughput_for_cluster: { data: [ @@ -22,8 +23,8 @@ describe('processPipelinesAPIResponse', () => { }, nodes_count_for_cluster: { data: [ - [1513721903, 3], - [1513722162, 2], + [1513721903, { 1: 5 }], + [1513722162, { 1: 10 }], ], }, }, @@ -32,96 +33,27 @@ describe('processPipelinesAPIResponse', () => { }; }); - it('normalizes the metric keys', () => { - processPipelinesAPIResponse(response, 'throughput_for_cluster', 'nodes_count_for_cluster').then( - processedResponse => { - expect(processedResponse.pipelines[0].metrics.throughput).to.eql( - response.pipelines[0].metrics.throughput_for_cluster - ); - expect(processedResponse.pipelines[0].metrics.nodesCount).to.eql( - response.pipelines[0].metrics.nodes_count_for_cluster - ); - } + it('normalizes the metric keys', async () => { + const processedResponse = await processPipelinesAPIResponse( + response, + 'throughput_for_cluster', + 'nodes_count_for_cluster' + ); + expect(processedResponse.pipelines[0].metrics.throughput).to.eql( + response.pipelines[0].metrics.throughput_for_cluster ); + expect(processedResponse.pipelines[0].metrics.nodesCount.data[0][0]).to.eql(1513721903); + expect(processedResponse.pipelines[0].metrics.nodesCount.data[0][1]).to.eql(5); + expect(processedResponse.pipelines[0].metrics.nodesCount.data[1][0]).to.eql(1513722162); + expect(processedResponse.pipelines[0].metrics.nodesCount.data[1][1]).to.eql(10); }); it('computes the latest metrics', () => { processPipelinesAPIResponse(response, 'throughput_for_cluster', 'nodes_count_for_cluster').then( processedResponse => { expect(processedResponse.pipelines[0].latestThroughput).to.eql(23); - expect(processedResponse.pipelines[0].latestNodesCount).to.eql(2); + expect(processedResponse.pipelines[0].latestNodesCount).to.eql(10); } ); }); }); - -describe('get_pipelines', () => { - let fetchPipelinesWithMetricsResult; - - describe('fetchPipelinesWithMetrics result contains no pipelines', () => { - beforeEach(() => { - fetchPipelinesWithMetricsResult = { - logstash_pipeline_throughput: [ - { - data: [], - }, - ], - logstash_pipeline_nodes_count: [ - { - data: [], - }, - ], - }; - }); - - it('returns an empty array', () => { - const result = handleGetPipelinesResponse(fetchPipelinesWithMetricsResult); - expect(result).to.eql([]); - }); - }); - - describe('fetchPipelinesWithMetrics result contains pipelines', () => { - beforeEach(() => { - fetchPipelinesWithMetricsResult = { - logstash_pipeline_throughput: [ - { - data: [[1513123151000, { apache_logs: 231, logstash_tweets: 34 }]], - }, - ], - logstash_pipeline_nodes_count: [ - { - data: [[1513123151000, { apache_logs: 3, logstash_tweets: 1 }]], - }, - ], - }; - }); - - it('returns the correct structure for a non-empty response', () => { - const result = handleGetPipelinesResponse(fetchPipelinesWithMetricsResult); - expect(result).to.eql([ - { - id: 'apache_logs', - metrics: { - logstash_pipeline_throughput: { - data: [[1513123151000, 231]], - }, - logstash_pipeline_nodes_count: { - data: [[1513123151000, 3]], - }, - }, - }, - { - id: 'logstash_tweets', - metrics: { - logstash_pipeline_throughput: { - data: [[1513123151000, 34]], - }, - logstash_pipeline_nodes_count: { - data: [[1513123151000, 1]], - }, - }, - }, - ]); - }); - }); -}); diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_logstash_for_clusters.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_logstash_for_clusters.js index d0de2c3f5df3a..55baa3cf10b50 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_logstash_for_clusters.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_logstash_for_clusters.js @@ -60,7 +60,7 @@ export function getLogstashForClusters(req, lsIndexPattern, clusters) { logstash_uuids: { terms: { field: 'logstash_stats.logstash.uuid', - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), }, aggs: { latest_report: { @@ -119,7 +119,7 @@ export function getLogstashForClusters(req, lsIndexPattern, clusters) { logstash_versions: { terms: { field: 'logstash_stats.logstash.version', - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), }, }, pipelines_nested: { @@ -135,7 +135,7 @@ export function getLogstashForClusters(req, lsIndexPattern, clusters) { queue_types: { terms: { field: 'logstash_stats.pipelines.queue.type', - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), }, aggs: { num_pipelines: { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_nodes.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_nodes.js index 93b70d7b79f0a..06696abdb031f 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_nodes.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_nodes.js @@ -31,7 +31,7 @@ export function getNodes(req, lsIndexPattern, { clusterUuid }) { const params = { index: lsIndexPattern, - size: config.get('xpack.monitoring.max_bucket_size'), // FIXME + size: config.get('monitoring.ui.max_bucket_size'), // FIXME ignoreUnavailable: true, body: { query: createQuery({ diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_paginated_pipelines.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_paginated_pipelines.js index c09df240d4f35..ffc7e9ce1d6c2 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_paginated_pipelines.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_paginated_pipelines.js @@ -7,7 +7,6 @@ import { get } from 'lodash'; import { filter } from '../pagination/filter'; import { getLogstashPipelineIds } from './get_pipeline_ids'; -import { handleGetPipelinesResponse } from './get_pipelines'; import { sortPipelines } from './sort_pipelines'; import { paginate } from '../pagination/paginate'; import { getMetrics } from '../details/get_metrics'; @@ -38,7 +37,7 @@ export async function getPaginatedPipelines( queryText ) { const config = req.server.config(); - const size = config.get('xpack.monitoring.max_bucket_size'); + const size = config.get('monitoring.ui.max_bucket_size'); const pipelines = await getLogstashPipelineIds( req, lsIndexPattern, @@ -51,19 +50,33 @@ export async function getPaginatedPipelines( // the necessary sort - we only need the last bucket of data so we // fetch the last two buckets of data (to ensure we have a single full bucekt), // then return the value from that last bucket - const metricSeriesData = await getMetrics( - req, - lsIndexPattern, - metricSet, - [], - { pageOfPipelines: pipelines }, - 2 - ); - const pipelineAggregationsData = handleGetPipelinesResponse( - metricSeriesData, - pipelines.map(p => p.id) + const metricSeriesData = Object.values( + await Promise.all( + pipelines.map(pipeline => { + return new Promise(async resolve => { + const data = await getMetrics( + req, + lsIndexPattern, + metricSet, + [], + { + pipeline, + }, + 2 + ); + + resolve({ + id: pipeline.id, + metrics: Object.keys(data).reduce((accum, metricName) => { + accum[metricName] = data[metricName][0]; + return accum; + }, {}), + }); + }); + }) + ) ); - for (const pipelineAggregationData of pipelineAggregationsData) { + for (const pipelineAggregationData of metricSeriesData) { for (const pipeline of pipelines) { if (pipelineAggregationData.id === pipeline.id) { for (const metric of metricSet) { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline.js index eeeffd74e91f7..35a4295de298b 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline.js @@ -111,7 +111,7 @@ export async function getPipeline(req, config, lsIndexPattern, clusterUuid, pipe }; // Determine metrics' timeseries interval based on version's timespan - const minIntervalSeconds = config.get('xpack.monitoring.min_interval_seconds'); + const minIntervalSeconds = config.get('monitoring.ui.min_interval_seconds'); const timeseriesInterval = calculateTimeseriesInterval( version.firstSeen, version.lastSeen, diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_ids.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_ids.js index d2cd5bef9d7ff..0773ab8948564 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_ids.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_ids.js @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import moment from 'moment'; -import { get, uniq } from 'lodash'; +import { get } from 'lodash'; import { createQuery } from '../create_query'; import { LogstashMetric } from '../metrics'; @@ -26,7 +26,7 @@ export async function getLogstashPipelineIds( index: logstashIndexPattern, size: 0, ignoreUnavailable: true, - filterPath: ['aggregations.nested_context.composite_data.buckets'], + filterPath: ['aggregations.nest.id.buckets'], body: { query: createQuery({ start, @@ -36,37 +36,28 @@ export async function getLogstashPipelineIds( filters, }), aggs: { - nested_context: { + nest: { nested: { path: 'logstash_stats.pipelines', }, aggs: { - composite_data: { - composite: { + id: { + terms: { + field: 'logstash_stats.pipelines.id', size, - sources: [ - { - id: { - terms: { - field: 'logstash_stats.pipelines.id', - }, - }, - }, - { - hash: { - terms: { - field: 'logstash_stats.pipelines.hash', - }, - }, - }, - { - ephemeral_id: { + }, + aggs: { + unnest: { + reverse_nested: {}, + aggs: { + nodes: { terms: { - field: 'logstash_stats.pipelines.ephemeral_id', + field: 'logstash_stats.logstash.uuid', + size, }, }, }, - ], + }, }, }, }, @@ -77,8 +68,8 @@ export async function getLogstashPipelineIds( const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); const response = await callWithRequest(req, 'search', params); - const data = get(response, 'aggregations.nested_context.composite_data.buckets', []).map( - bucket => bucket.key - ); - return uniq(data, item => item.id); + return get(response, 'aggregations.nest.id.buckets', []).map(bucket => ({ + id: bucket.key, + nodeIds: get(bucket, 'unnest.nodes.buckets', []).map(item => item.key), + })); } diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_stats_aggregation.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_stats_aggregation.js index 1858674a01b86..d9c03819b0098 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_stats_aggregation.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_stats_aggregation.js @@ -171,7 +171,7 @@ export function getPipelineStatsAggregation( logstashIndexPattern, pipelineId, version, - config.get('xpack.monitoring.max_bucket_size'), + config.get('monitoring.ui.max_bucket_size'), callWithRequest, req ); diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_versions.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_versions.js index 7dfa8d4a163ce..7521389c379ea 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_versions.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_versions.js @@ -37,7 +37,7 @@ function fetchPipelineVersions(...args) { by_pipeline_hash: { terms: { field: 'logstash_stats.pipelines.hash', - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), order: { 'path_to_root>first_seen': 'desc' }, }, aggs: { diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_vertex.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_vertex.js index 49c2dff2d6080..134dd88b36ce6 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_vertex.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_vertex.js @@ -130,7 +130,7 @@ export async function getPipelineVertex( }; // Determine metrics' timeseries interval based on version's timespan - const minIntervalSeconds = config.get('xpack.monitoring.min_interval_seconds'); + const minIntervalSeconds = config.get('monitoring.ui.min_interval_seconds'); const timeseriesInterval = calculateTimeseriesInterval( version.firstSeen, version.lastSeen, diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_vertex_stats_aggregation.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_vertex_stats_aggregation.js index c91182188b213..425ca5731926c 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_vertex_stats_aggregation.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipeline_vertex_stats_aggregation.js @@ -216,7 +216,7 @@ export function getPipelineVertexStatsAggregation( version, vertexId, timeSeriesIntervalInSeconds, - config.get('xpack.monitoring.max_bucket_size'), + config.get('monitoring.ui.max_bucket_size'), callWithRequest, req ); diff --git a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipelines.js b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipelines.js index 7b52a26f0e80d..d634170bdd9fe 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipelines.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/logstash/get_pipelines.js @@ -3,66 +3,10 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { cloneDeep, last, omit } from 'lodash'; +import { cloneDeep, last } from 'lodash'; import { checkParam } from '../error_missing_required'; import { getMetrics } from '../details/get_metrics'; -export function handleGetPipelinesResponse(response, exclusivePipelineIds) { - const pipelinesById = {}; - - const metrics = Object.keys(response); - metrics.forEach(metric => { - response[metric][0].data.forEach(([x, y]) => { - const pipelineIds = Object.keys(y); - pipelineIds.forEach(pipelineId => { - if (exclusivePipelineIds && !exclusivePipelineIds.includes(pipelineId)) { - return; - } - // Create new pipeline object if necessary - if (!pipelinesById.hasOwnProperty(pipelineId)) { - pipelinesById[pipelineId] = { - metrics: {}, - }; - } - const pipeline = pipelinesById[pipelineId]; - - // Create new metric object in pipeline object if necessary - if (!pipeline.metrics.hasOwnProperty(metric)) { - // Clone the metric object from the response so we don't accidentally overwrite it - // in the code further below. Also, reset data to empty array because we only want - // to keep data "y" values specific to this pipeline - pipeline.metrics[metric] = { - ...omit(response[metric][0], 'data'), - data: [], - }; - } - - pipeline.metrics[metric].data.push([x, y[pipelineId]]); - }); - }); - }); - - // Convert pipelinesById map to array and preserve sorting - const pipelines = []; - if (exclusivePipelineIds) { - for (const exclusivePipelineId of exclusivePipelineIds) { - pipelines.push({ - id: exclusivePipelineId, - ...pipelinesById[exclusivePipelineId], - }); - } - } else { - Object.keys(pipelinesById).forEach(pipelineId => { - pipelines.push({ - id: pipelineId, - ...pipelinesById[pipelineId], - }); - }); - } - - return pipelines; -} - export async function processPipelinesAPIResponse( response, throughputMetricKey, @@ -76,7 +20,13 @@ export async function processPipelinesAPIResponse( processedResponse.pipelines.forEach(pipeline => { pipeline.metrics = { throughput: pipeline.metrics[throughputMetricKey], - nodesCount: pipeline.metrics[nodesCountMetricKey], + nodesCount: { + ...pipeline.metrics[nodesCountMetricKey], + data: pipeline.metrics[nodesCountMetricKey].data.map(item => [ + item[0], + item[1][pipeline.id], + ]), + }, }; pipeline.latestThroughput = last(pipeline.metrics.throughput.data)[1]; @@ -86,24 +36,29 @@ export async function processPipelinesAPIResponse( return processedResponse; } -export async function getPipelines( - req, - logstashIndexPattern, - pipelineIds, - metricSet, - metricOptions = {} -) { +export async function getPipelines(req, logstashIndexPattern, pipelines, metricSet) { checkParam(logstashIndexPattern, 'logstashIndexPattern in logstash/getPipelines'); checkParam(metricSet, 'metricSet in logstash/getPipelines'); const filters = []; - const metricsResponse = await getMetrics( - req, - logstashIndexPattern, - metricSet, - filters, - metricOptions + const metricsResponse = await Promise.all( + pipelines.map(pipeline => { + return new Promise(async resolve => { + const data = await getMetrics(req, logstashIndexPattern, metricSet, filters, { + pipeline, + }); + + resolve({ + id: pipeline.id, + metrics: Object.keys(data).reduce((accum, metricName) => { + accum[metricName] = data[metricName][0]; + return accum; + }, {}), + }); + }); + }) ); - return handleGetPipelinesResponse(metricsResponse, pipelineIds); + + return Object.values(metricsResponse); } diff --git a/x-pack/legacy/plugins/monitoring/server/lib/metrics/__test__/__snapshots__/metrics.test.js.snap b/x-pack/legacy/plugins/monitoring/server/lib/metrics/__test__/__snapshots__/metrics.test.js.snap index 2970a9e3ec9ca..595ff3ccfced4 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/metrics/__test__/__snapshots__/metrics.test.js.snap +++ b/x-pack/legacy/plugins/monitoring/server/lib/metrics/__test__/__snapshots__/metrics.test.js.snap @@ -3020,8 +3020,7 @@ Object { }, "logstash_cluster_pipeline_throughput": LogstashPipelineThroughputMetric { "app": "logstash", - "calculation": [Function], - "derivative": false, + "derivative": true, "description": "Number of events emitted per second by the Logstash pipeline at the outputs stage.", "field": "logstash_stats.pipelines.events.out", "format": "0,0.[00]", @@ -3296,8 +3295,7 @@ Object { }, "logstash_node_pipeline_throughput": LogstashPipelineThroughputMetric { "app": "logstash", - "calculation": [Function], - "derivative": false, + "derivative": true, "description": "Number of events emitted per second by the Logstash pipeline at the outputs stage.", "field": "logstash_stats.pipelines.events.out", "format": "0,0.[00]", diff --git a/x-pack/legacy/plugins/monitoring/server/lib/metrics/logstash/classes.js b/x-pack/legacy/plugins/monitoring/server/lib/metrics/logstash/classes.js index eddcfabe83b1b..1f45d76ecff28 100644 --- a/x-pack/legacy/plugins/monitoring/server/lib/metrics/logstash/classes.js +++ b/x-pack/legacy/plugins/monitoring/server/lib/metrics/logstash/classes.js @@ -250,59 +250,45 @@ export class LogstashPipelineThroughputMetric extends LogstashMetric { constructor(opts) { super({ ...opts, - derivative: false, + derivative: true, }); - this.getDateHistogramSubAggs = ({ pageOfPipelines }) => ({ - pipelines_nested: { - nested: { - path: 'logstash_stats.pipelines', + this.getDateHistogramSubAggs = ({ pipeline }) => { + return { + metric_deriv: { + derivative: { + buckets_path: 'sum', + gap_policy: 'skip', + unit: NORMALIZED_DERIVATIVE_UNIT, + }, }, - aggs: { - by_pipeline_id: { - terms: { - field: 'logstash_stats.pipelines.id', - size: 1000, - include: pageOfPipelines.map(pipeline => pipeline.id), - }, - aggs: { - throughput: { - sum_bucket: { - buckets_path: 'by_pipeline_hash>throughput', - }, + sum: { + sum_bucket: { + buckets_path: 'by_node_id>nest>pipeline>events_stats', + }, + }, + by_node_id: { + terms: { + field: 'logstash_stats.logstash.uuid', + size: 1000, + include: pipeline.uuids, + }, + aggs: { + nest: { + nested: { + path: 'logstash_stats.pipelines', }, - by_pipeline_hash: { - terms: { - field: 'logstash_stats.pipelines.hash', - size: 1000, - include: pageOfPipelines.map(pipeline => pipeline.hash), - }, - aggs: { - throughput: { - sum_bucket: { - buckets_path: 'by_ephemeral_id>throughput', + aggs: { + pipeline: { + filter: { + term: { + 'logstash_stats.pipelines.id': pipeline.id, }, }, - by_ephemeral_id: { - terms: { - field: 'logstash_stats.pipelines.ephemeral_id', - size: 1000, - include: pageOfPipelines.map(pipeline => pipeline.ephemeral_id), - }, - aggs: { - events_stats: { - stats: { - field: this.field, - }, - }, - throughput: { - bucket_script: { - script: 'params.max - params.min', - buckets_path: { - min: 'events_stats.min', - max: 'events_stats.max', - }, - }, + aggs: { + events_stats: { + max: { + field: this.field, }, }, }, @@ -311,19 +297,7 @@ export class LogstashPipelineThroughputMetric extends LogstashMetric { }, }, }, - }, - }); - - this.calculation = (bucket, _key, _metric, bucketSizeInSeconds) => { - const pipelineThroughputs = {}; - const pipelineBuckets = _.get(bucket, 'pipelines_nested.by_pipeline_id.buckets', []); - pipelineBuckets.forEach(pipelineBucket => { - pipelineThroughputs[pipelineBucket.key] = bucketSizeInSeconds - ? _.get(pipelineBucket, 'throughput.value') / bucketSizeInSeconds - : undefined; - }); - - return pipelineThroughputs; + }; }; } } diff --git a/x-pack/legacy/plugins/monitoring/server/plugin.js b/x-pack/legacy/plugins/monitoring/server/plugin.js index 163bc43945be1..ef346e95ad075 100644 --- a/x-pack/legacy/plugins/monitoring/server/plugin.js +++ b/x-pack/legacy/plugins/monitoring/server/plugin.js @@ -48,7 +48,7 @@ export class Plugin { /* * End-user-facing services */ - const uiEnabled = config.get('xpack.monitoring.ui.enabled'); + const uiEnabled = config.get('monitoring.ui.enabled'); if (uiEnabled) { await instantiateClient({ @@ -98,7 +98,7 @@ export class Plugin { kbnServerStatus: kbnServer.status, kbnServerVersion: kbnServer.version, }); - const kibanaCollectionEnabled = config.get('xpack.monitoring.kibana.collection.enabled'); + const kibanaCollectionEnabled = config.get('monitoring.kibana.collection.enabled'); if (kibanaCollectionEnabled) { /* @@ -125,14 +125,12 @@ export class Plugin { core.injectUiAppVars('monitoring', () => { const config = core.config(); return { - maxBucketSize: config.get('xpack.monitoring.max_bucket_size'), - minIntervalSeconds: config.get('xpack.monitoring.min_interval_seconds'), + maxBucketSize: config.get('monitoring.ui.max_bucket_size'), + minIntervalSeconds: config.get('monitoring.ui.min_interval_seconds'), kbnIndex: config.get('kibana.index'), - showLicenseExpiration: config.get('xpack.monitoring.show_license_expiration'), - showCgroupMetricsElasticsearch: config.get( - 'xpack.monitoring.ui.container.elasticsearch.enabled' - ), - showCgroupMetricsLogstash: config.get('xpack.monitoring.ui.container.logstash.enabled'), // Note, not currently used, but see https://github.com/elastic/x-pack-kibana/issues/1559 part 2 + showLicenseExpiration: config.get('monitoring.ui.show_license_expiration'), + showCgroupMetricsElasticsearch: config.get('monitoring.ui.container.elasticsearch.enabled'), + showCgroupMetricsLogstash: config.get('monitoring.ui.container.logstash.enabled'), // Note, not currently used, but see https://github.com/elastic/x-pack-kibana/issues/1559 part 2 }; }); } diff --git a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.js b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.js index 2d4bded9fc4c8..fcdf4ad8a706c 100644 --- a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.js +++ b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.js @@ -26,7 +26,7 @@ function getBucketScript(max, min) { function buildRequest(req, config, esIndexPattern) { const min = moment.utc(req.payload.timeRange.min).valueOf(); const max = moment.utc(req.payload.timeRange.max).valueOf(); - const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); + const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); const aggs = { ops_synced_max: { max: { diff --git a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/elasticsearch/node_detail.js b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/elasticsearch/node_detail.js index 10226d74ed001..25ead723e3ddb 100644 --- a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/elasticsearch/node_detail.js +++ b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/elasticsearch/node_detail.js @@ -61,7 +61,7 @@ export function esNodeRoute(server) { metricSet = metricSetOverview; // set the cgroup option if needed const showCgroupMetricsElasticsearch = config.get( - 'xpack.monitoring.ui.container.elasticsearch.enabled' + 'monitoring.ui.container.elasticsearch.enabled' ); const metricCpu = metricSet.find(m => m.name === 'node_cpu_metric'); if (showCgroupMetricsElasticsearch) { diff --git a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/node.js b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/node.js index d5ce9d1686f8a..bd3ae5f5c2679 100644 --- a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/node.js +++ b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/node.js @@ -60,9 +60,7 @@ export function logstashNodeRoute(server) { } else { metricSet = metricSetOverview; // set the cgroup option if needed - const showCgroupMetricsLogstash = config.get( - 'xpack.monitoring.ui.container.logstash.enabled' - ); + const showCgroupMetricsLogstash = config.get('monitoring.ui.container.logstash.enabled'); const metricCpu = metricSet.find(m => m.name === 'logstash_node_cpu_metric'); if (showCgroupMetricsLogstash) { metricCpu.keys = ['logstash_node_cgroup_quota_as_cpu_utilization']; diff --git a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipeline_ids.js b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipeline_ids.js index c5fd76487cca1..93330880babcc 100644 --- a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipeline_ids.js +++ b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipeline_ids.js @@ -36,7 +36,7 @@ export function logstashClusterPipelineIdsRoute(server) { const { ccs } = req.payload; const clusterUuid = req.params.clusterUuid; const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs); - const size = config.get('xpack.monitoring.max_bucket_size'); + const size = config.get('monitoring.ui.max_bucket_size'); try { const pipelines = await getLogstashPipelineIds(req, lsIndexPattern, { clusterUuid }, size); diff --git a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipelines.js b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipelines.js index 1c796ced96f9b..0839bd4800329 100644 --- a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipelines.js +++ b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/cluster_pipelines.js @@ -79,21 +79,8 @@ export function logstashClusterPipelinesRoute(server) { queryText ); - // Just the IDs for the rest - const pipelineIds = pageOfPipelines.map(pipeline => pipeline.id); - - const metricOptions = { - pageOfPipelines, - }; - try { - const pipelineData = await getPipelines( - req, - lsIndexPattern, - pipelineIds, - metricSet, - metricOptions - ); + const pipelineData = await getPipelines(req, lsIndexPattern, pageOfPipelines, metricSet); const response = await processPipelinesAPIResponse( { pipelines: pipelineData, diff --git a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/node_pipelines.js b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/node_pipelines.js index a8cad480b9c37..604cc86b81b58 100644 --- a/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/node_pipelines.js +++ b/x-pack/legacy/plugins/monitoring/server/routes/api/v1/logstash/pipelines/node_pipelines.js @@ -78,22 +78,8 @@ export function logstashNodePipelinesRoute(server) { sort, queryText ); - - // Just the IDs for the rest - const pipelineIds = pageOfPipelines.map(pipeline => pipeline.id); - - const metricOptions = { - pageOfPipelines, - }; - try { - const pipelineData = await getPipelines( - req, - lsIndexPattern, - pipelineIds, - metricSet, - metricOptions - ); + const pipelineData = await getPipelines(req, lsIndexPattern, pageOfPipelines, metricSet); const response = await processPipelinesAPIResponse( { pipelines: pipelineData, diff --git a/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts b/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts index fc85cbe442ddf..4738ab5b8af83 100644 --- a/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts +++ b/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts @@ -40,7 +40,7 @@ export function fetchClusterUuids({ server, callCluster, start, end }: StatsColl cluster_uuids: { terms: { field: 'cluster_uuid', - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), }, }, }, diff --git a/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_es_stats.js b/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_es_stats.js index 8e5a59361e52f..52d34258b5fa4 100644 --- a/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_es_stats.js +++ b/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_es_stats.js @@ -31,7 +31,7 @@ export function fetchElasticsearchStats(server, callCluster, clusterUuids) { const config = server.config(); const params = { index: INDEX_PATTERN_ELASTICSEARCH, - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), ignoreUnavailable: true, filterPath: [ 'hits.hits._source.cluster_uuid', diff --git a/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_high_level_stats.js b/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_high_level_stats.js index 2632a8f6e041d..b87f632308e4d 100644 --- a/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_high_level_stats.js +++ b/x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_high_level_stats.js @@ -217,7 +217,7 @@ export async function fetchHighLevelStats(server, callCluster, clusterUuids, sta const params = { index: getIndexPatternForStackProduct(product), - size: config.get('xpack.monitoring.max_bucket_size'), + size: config.get('monitoring.ui.max_bucket_size'), headers: { 'X-QUERY-SOURCE': TELEMETRY_QUERY_SOURCE, }, diff --git a/x-pack/legacy/plugins/monitoring/ui_exports.js b/x-pack/legacy/plugins/monitoring/ui_exports.js index 2b5ea21a2bb45..9251deb673bd1 100644 --- a/x-pack/legacy/plugins/monitoring/ui_exports.js +++ b/x-pack/legacy/plugins/monitoring/ui_exports.js @@ -32,7 +32,7 @@ export const getUiExports = () => ({ injectDefaultVars(server) { const config = server.config(); return { - monitoringUiEnabled: config.get('xpack.monitoring.ui.enabled'), + monitoringUiEnabled: config.get('monitoring.ui.enabled'), }; }, hacks: ['plugins/monitoring/hacks/toggle_app_link_in_nav'], diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/ecs_mapping.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/ecs_mapping.json index 06edf94484af3..5faa53db51101 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/ecs_mapping.json +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/ecs_mapping.json @@ -37,6 +37,12 @@ "organization": { "properties": { "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -58,6 +64,12 @@ "organization": { "properties": { "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -149,6 +161,12 @@ "type": "keyword" }, "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -177,6 +195,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -273,6 +297,12 @@ "organization": { "properties": { "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -364,6 +394,12 @@ "type": "keyword" }, "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -392,6 +428,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -422,7 +464,8 @@ "ignore_above": 1024, "type": "keyword" } - } + }, + "type": "object" }, "header_flags": { "ignore_above": 1024, @@ -501,6 +544,12 @@ }, "stack_trace": { "doc_values": false, + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "index": false, "type": "keyword" @@ -546,6 +595,9 @@ "ignore_above": 1024, "type": "keyword" }, + "ingested": { + "type": "date" + }, "kind": { "ignore_above": 1024, "type": "keyword" @@ -598,6 +650,10 @@ "accessed": { "type": "date" }, + "attributes": { + "ignore_above": 1024, + "type": "keyword" + }, "created": { "type": "date" }, @@ -612,6 +668,10 @@ "ignore_above": 1024, "type": "keyword" }, + "drive_letter": { + "ignore_above": 1, + "type": "keyword" + }, "extension": { "ignore_above": 1024, "type": "keyword" @@ -664,6 +724,12 @@ "type": "keyword" }, "path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -671,6 +737,12 @@ "type": "long" }, "target_path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -761,6 +833,10 @@ "ignore_above": 1024, "type": "keyword" }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, "geo": { "properties": { "city_name": { @@ -822,6 +898,12 @@ "type": "keyword" }, "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -830,6 +912,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -861,6 +949,12 @@ "type": "keyword" }, "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -889,6 +983,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -906,6 +1006,12 @@ "type": "long" }, "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -932,6 +1038,12 @@ "type": "long" }, "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -1016,7 +1128,8 @@ } } } - } + }, + "type": "object" } } }, @@ -1128,6 +1241,12 @@ "type": "keyword" }, "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1136,6 +1255,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1178,6 +1303,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -1190,6 +1321,12 @@ "type": "keyword" }, "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1198,6 +1335,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1217,6 +1360,10 @@ "ignore_above": 1024, "type": "keyword" }, + "build_version": { + "ignore_above": 1024, + "type": "keyword" + }, "checksum": { "ignore_above": 1024, "type": "keyword" @@ -1244,9 +1391,17 @@ "ignore_above": 1024, "type": "keyword" }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, "size": { "type": "long" }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, "version": { "ignore_above": 1024, "type": "keyword" @@ -1259,10 +1414,32 @@ "ignore_above": 1024, "type": "keyword" }, + "args_count": { + "type": "long" + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, + "exit_code": { + "type": "long" + }, "hash": { "properties": { "md5": { @@ -1284,9 +1461,105 @@ } }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, + "parent": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, "pgid": { "type": "long" }, @@ -1311,6 +1584,12 @@ } }, "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1318,6 +1597,48 @@ "type": "long" }, "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "registry": { + "properties": { + "data": { + "properties": { + "bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "strings": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hive": { + "ignore_above": 1024, + "type": "keyword" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { "ignore_above": 1024, "type": "keyword" } @@ -1325,8 +1646,52 @@ }, "related": { "properties": { + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, "ip": { "type": "ip" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -1344,6 +1709,12 @@ "organization": { "properties": { "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -1435,6 +1806,12 @@ "type": "keyword" }, "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1463,6 +1840,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -1520,6 +1903,12 @@ "organization": { "properties": { "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -1611,6 +2000,12 @@ "type": "keyword" }, "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1639,6 +2034,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -1679,6 +2080,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1690,6 +2097,136 @@ } } }, + "tls": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "client": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "supported_ciphers": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "established": { + "type": "boolean" + }, + "next_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "resumed": { + "type": "boolean" + }, + "server": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3s": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "version_protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "trace": { "properties": { "id": { @@ -1721,10 +2258,22 @@ "type": "keyword" }, "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1772,6 +2321,12 @@ "type": "keyword" }, "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1800,6 +2355,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -1820,6 +2381,12 @@ "type": "keyword" }, "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1830,6 +2397,12 @@ "type": "keyword" }, "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1838,6 +2411,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, @@ -1856,7 +2435,85 @@ "type": "keyword" } } + }, + "vulnerability": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "classification": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "enumeration": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "report_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner": { + "properties": { + "vendor": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "score": { + "properties": { + "base": { + "type": "float" + }, + "environmental": { + "type": "float" + }, + "temporal": { + "type": "float" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } } } + }, + "order": 1, + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": 10000 + } + }, + "refresh_interval": "5s" + } } } diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts index 4b04cb257d4ac..e4f612a14832b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts @@ -21,9 +21,11 @@ import { jest.mock('../../rules/get_prepackaged_rules', () => { return { - getPrepackagedRules: () => { + getPrepackagedRules: (): PrepackagedRules[] => { return [ { + tags: [], + immutable: true, rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -43,6 +45,7 @@ jest.mock('../../rules/get_prepackaged_rules', () => { }); import { addPrepackedRulesRoute } from './add_prepackaged_rules_route'; +import { PrepackagedRules } from '../../types'; describe('add_prepackaged_rules_route', () => { let { server, alertsClient, actionsClient, elasticsearch } = createMockServer(); @@ -57,7 +60,7 @@ describe('add_prepackaged_rules_route', () => { describe('status codes with actionClient and alertClient', () => { test('returns 200 when creating a with a valid actionClient and alertClient', async () => { - alertsClient.find.mockResolvedValue(getFindResult()); + alertsClient.find.mockResolvedValue(getFindResultWithSingleHit()); alertsClient.get.mockResolvedValue(getResult()); actionsClient.create.mockResolvedValue(createActionResult()); alertsClient.create.mockResolvedValue(getResult()); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts index 060659d0e1897..c631ed8f784ab 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -105,7 +105,7 @@ export const createCreateRulesRoute = (server: ServerFacade): Hapi.ServerRoute = timelineTitle, meta, filters, - ruleId: ruleId != null ? ruleId : uuid.v4(), + ruleId: ruleId ?? uuid.v4(), index, interval, maxSignals, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/add_tags.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/add_tags.test.ts index 8bfd9bef63733..70b705309b882 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/add_tags.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/add_tags.test.ts @@ -8,46 +8,6 @@ import { addTags } from './add_tags'; import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; describe('add_tags', () => { - test('if given a null everything this returns a new array for tags', () => { - const tags = addTags(null, null, null); - expect(tags).toEqual([]); - }); - - test('if given a undefined everything this returns a new array for tags', () => { - const tags = addTags(undefined, undefined, undefined); - expect(tags).toEqual([]); - }); - - test('it should add a rule id as an internal structure to a single tag', () => { - const tags = addTags(['tag 1'], 'rule-1', null); - expect(tags).toEqual(['tag 1', `${INTERNAL_RULE_ID_KEY}:rule-1`]); - }); - - test('it should add a rule id as an internal structure to a single tag if the input tags is null', () => { - const tags = addTags(null, 'rule-1', null); - expect(tags).toEqual([`${INTERNAL_RULE_ID_KEY}:rule-1`]); - }); - - test('it should add a rule id as an internal structure to two tags', () => { - const tags = addTags(['tag 1', 'tag 2'], 'rule-1', null); - expect(tags).toEqual(['tag 1', 'tag 2', `${INTERNAL_RULE_ID_KEY}:rule-1`]); - }); - - test('it should add a rule id as an internal structure with empty tags', () => { - const tags = addTags([], 'rule-1', null); - expect(tags).toEqual([`${INTERNAL_RULE_ID_KEY}:rule-1`]); - }); - - test('it should add a immutable true as an internal structure with empty tags', () => { - const tags = addTags([], null, true); - expect(tags).toEqual([`${INTERNAL_IMMUTABLE_KEY}:true`]); - }); - - test('it should add a immutable false as an internal structure with empty tags', () => { - const tags = addTags([], null, false); - expect(tags).toEqual([`${INTERNAL_IMMUTABLE_KEY}:false`]); - }); - test('it should add a rule id as an internal structure with immutable true', () => { const tags = addTags([], 'rule-1', true); expect(tags).toEqual([`${INTERNAL_RULE_ID_KEY}:rule-1`, `${INTERNAL_IMMUTABLE_KEY}:true`]); @@ -58,18 +18,22 @@ describe('add_tags', () => { expect(tags).toEqual([`${INTERNAL_RULE_ID_KEY}:rule-1`, `${INTERNAL_IMMUTABLE_KEY}:false`]); }); - test('it should add not add an internal structure if only a tag is given', () => { - const tags = addTags(['tag 1'], undefined, null); - expect(tags).toEqual(['tag 1']); - }); - - test('it should add not add an internal structure if everything is null', () => { - const tags = addTags(['tag 1'], null, null); - expect(tags).toEqual(['tag 1']); + test('it should not allow duplicate tags to be created', () => { + const tags = addTags(['tag-1', 'tag-1'], 'rule-1', false); + expect(tags).toEqual([ + 'tag-1', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:false`, + ]); }); - test('it should add not add an internal structure if everything is undefined', () => { - const tags = addTags(['tag 1'], undefined, undefined); - expect(tags).toEqual(['tag 1']); + test('it should not allow duplicate internal tags to be created when called two times in a row', () => { + const tags1 = addTags(['tag-1'], 'rule-1', false); + const tags2 = addTags(tags1, 'rule-1', false); + expect(tags2).toEqual([ + 'tag-1', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:false`, + ]); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/add_tags.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/add_tags.ts index f33f6ada83190..a1f240495045c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/add_tags.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/add_tags.ts @@ -6,23 +6,12 @@ import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; -export const addTags = ( - tags: string[] | null | undefined, - ruleId: string | null | undefined, - immutable: boolean | null | undefined -): string[] => { - const defaultedTags = tags != null ? tags : []; - if (ruleId != null && immutable != null) { - return [ - ...defaultedTags, +export const addTags = (tags: string[], ruleId: string, immutable: boolean): string[] => { + return Array.from( + new Set([ + ...tags, `${INTERNAL_RULE_ID_KEY}:${ruleId}`, `${INTERNAL_IMMUTABLE_KEY}:${immutable}`, - ]; - } else if (ruleId != null && immutable == null) { - return [...defaultedTags, `${INTERNAL_RULE_ID_KEY}:${ruleId}`]; - } else if (ruleId == null && immutable != null) { - return [...defaultedTags, `${INTERNAL_IMMUTABLE_KEY}:${immutable}`]; - } else { - return defaultedTags; - } + ]) + ); }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts index d65f5f84c6d64..30e8c4dbf9d88 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts @@ -5,7 +5,7 @@ */ import { APP_ID, SIGNALS_ID } from '../../../../common/constants'; -import { RuleParams } from './types'; +import { CreateRuleParams } from './types'; import { addTags } from './add_tags'; export const createRules = ({ @@ -37,7 +37,7 @@ export const createRules = ({ type, references, version, -}: RuleParams) => { +}: CreateRuleParams) => { return alertsClient.create({ data: { name, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts index 7b38794467df5..1b13a010bbea3 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts @@ -163,7 +163,7 @@ export type DeleteRuleParams = Clients & { ruleId: string | undefined | null; }; -export type RuleParams = RuleAlertParams & Clients; +export type CreateRuleParams = Omit & { ruleId: string } & Clients; export interface ReadRuleParams { alertsClient: AlertsClient; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts index e37e899c7de51..8234b931ad89a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/update_rules.ts @@ -198,11 +198,7 @@ export const updateRules = async ({ return alertsClient.update({ id: rule.id, data: { - tags: addTags( - tags != null ? tags : rule.tags, // Add tags as an update if it exists, otherwise re-use the older tags - rule.params.ruleId, - immutable != null ? immutable : rule.params.immutable // Add new one if it exists, otherwise re-use old one - ), + tags: addTags(tags ?? rule.tags, rule.params.ruleId, immutable ?? rule.params.immutable), name: calculateName({ updatedName: name, originalName: rule.name }), schedule: { interval: calculateInterval(interval, rule.schedule.interval),