From 678b026f34ea333e23a83b59b95b2f388076f1dc Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Mon, 24 Dec 2018 08:50:03 +0100 Subject: [PATCH 1/3] Increase max compilation rate for Filebeat modules tests Module tests started to fail with the following message: ``` Too many dynamic script compilations within, max: [75/5m]; please use indexed, or scripts with parameters instead; this limit can be changed by the [script.max_compilations_rate] setting ``` The reason for this is (I think) that we started to use scripts more heavily in our pipelines so we hit this limit. For the tests we can easily increase the limit to solve this. In production this could still be an issue when all pipelines are loaded at the same time and we need to find a solution here. In parallel we should also investigate if there are some non parameterised scripts which get recompiled every time. --- filebeat/tests/system/test_modules.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 8f54b152cb2..a759bd8081e 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -67,6 +67,14 @@ def init(self): self.index_name = "test-filebeat-modules" + body = { + "transient": { + "script.max_compilations_rate": "100/1m" + } + } + + self.es.transport.perform_request('PUT', "/_cluster/settings", body=body) + @parameterized.expand(load_fileset_test_cases) @unittest.skipIf(not INTEGRATION_TESTS, "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") From baea398d44976853ce741be38113c98b7de019ab Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Thu, 27 Dec 2018 09:17:44 +0100 Subject: [PATCH 2/3] increase --- filebeat/tests/system/test_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index a759bd8081e..154b612c275 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -69,7 +69,7 @@ def init(self): body = { "transient": { - "script.max_compilations_rate": "100/1m" + "script.max_compilations_rate": "1000/1m" } } From 751a565ddbee07795019e792a34fd8d7eafae499 Mon Sep 17 00:00:00 2001 From: beats-jenkins Date: Thu, 27 Dec 2018 09:25:04 +0100 Subject: [PATCH 3/3] increase limit to 1000 and fix non updated postgres files --- ...gresql-9.6-new-timestamp.log-expected.json | 59 +++++++++---------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json index 3a3fb62a734..5f10dd18f66 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json @@ -3,74 +3,69 @@ "@timestamp": "2017-07-31T13:36:43.000Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", + "log.level": "LOG", "log.offset": 0, - "message": "2017-07-31 13:36:43 EST [835-1] LOG: autovacuum launcher started", - "postgresql.log.core_id": "1", - "postgresql.log.level": "LOG", - "postgresql.log.message": "autovacuum launcher started", - "postgresql.log.thread_id": "835", + "message": "autovacuum launcher started", + "postgresql.log.core_id": 1, "postgresql.log.timestamp": "2017-07-31 13:36:43", - "postgresql.log.timezone": "EST" + "process.pid": 835 }, { "@timestamp": "2017-07-31T13:36:44.000Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", + "log.level": "LOG", "log.offset": 66, - "message": "2017-07-31 13:36:44 EST [832-1] LOG: checkpoints are occurring too frequently (25 seconds apart)", - "postgresql.log.core_id": "1", - "postgresql.log.level": "LOG", - "postgresql.log.message": "checkpoints are occurring too frequently (25 seconds apart)", - "postgresql.log.thread_id": "832", + "message": "checkpoints are occurring too frequently (25 seconds apart)", + "postgresql.log.core_id": 1, "postgresql.log.timestamp": "2017-07-31 13:36:44", - "postgresql.log.timezone": "EST" + "process.pid": 832 }, { "@timestamp": "2017-07-31T13:46:02.000Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", + "log.level": "HINT", "log.offset": 164, - "message": "2017-07-31 13:46:02 EST [832-2] HINT: Consider increasing the configuration parameter \"max_wal_size\".", - "postgresql.log.core_id": "2", - "postgresql.log.level": "HINT", - "postgresql.log.message": "Consider increasing the configuration parameter \"max_wal_size\".", - "postgresql.log.thread_id": "832", + "message": "Consider increasing the configuration parameter \"max_wal_size\".", + "postgresql.log.core_id": 2, "postgresql.log.timestamp": "2017-07-31 13:46:02", - "postgresql.log.timezone": "EST" + "process.pid": 832 }, { "@timestamp": "2017-07-31T13:46:23.000Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", + "log.level": "FATAL", "log.offset": 267, - "message": "2017-07-31 13:46:23 EST [768-1] postgres@postgres FATAL: the database system is starting up", - "postgresql.log.core_id": "1", + "message": "the database system is starting up", + "postgresql.log.core_id": 1, "postgresql.log.database": "postgres", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "the database system is starting up", - "postgresql.log.thread_id": "768", "postgresql.log.timestamp": "2017-07-31 13:46:23", - "postgresql.log.timezone": "EST", - "postgresql.log.user": "postgres" + "process.pid": 768, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:46:55.000Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", + "log.level": "FATAL", "log.offset": 360, - "message": "2017-07-31 13:46:55 EST [771-1] postgres@postgres FATAL: the database system is starting up", - "postgresql.log.core_id": "1", + "message": "the database system is starting up", + "postgresql.log.core_id": 1, "postgresql.log.database": "postgres", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "the database system is starting up", - "postgresql.log.thread_id": "771", "postgresql.log.timestamp": "2017-07-31 13:46:55", - "postgresql.log.timezone": "EST", - "postgresql.log.user": "postgres" + "process.pid": 771, + "user.name": "postgres" } ] \ No newline at end of file