Fix Filebeat tests with new region_iso_code field (#7678)

In elastic/elasticsearch#31669 the field `region_iso_code` was added to the geoip processor. Because of this test broke with the most recent release of Elasticsearch as the events contain an undocumented field.
elastic · Jul 23, 2018 · e2ddff7 · e2ddff7
1 parent b2fb25d
commit e2ddff7
Show file tree

Hide file tree

Showing 17 changed files with 123 additions and 18 deletions.
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -305,6 +305,16 @@ type: keyword
 The city name.
 
 
+--
+
+*`apache2.access.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [float]
@@ -556,6 +566,16 @@ type: geo_point
 The longitude and latitude.
 
 
+--
+
+*`auditd.log.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [[exported-fields-beat]]
@@ -2000,6 +2020,16 @@ type: keyword
 The city name.
 
 
+--
+
+*`iis.access.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [float]
@@ -2166,6 +2196,16 @@ type: keyword
 The city name.
 
 
+--
+
+*`iis.error.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [[exported-fields-kafka]]
@@ -3353,6 +3393,16 @@ type: keyword
 The city name.
 
 
+--
+
+*`nginx.access.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [float]
@@ -3890,6 +3940,16 @@ type: geo_point
 The longitude and latitude.
 
 
+--
+
+*`system.auth.ssh.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [float]
@@ -4333,6 +4393,16 @@ type: keyword
 The city name.
 
 
+--
+
+*`traefik.access.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 *`traefik.access.request_count`*::

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/apache2/access/_meta/fields.yml b/filebeat/module/apache2/access/_meta/fields.yml
@@ -112,4 +112,7 @@
           type: keyword
           description: >
             The city name.
-
+        - name: region_iso_code
+          type: keyword
+          description: >
+            Region ISO code.
diff --git a/filebeat/module/auditd/log/_meta/fields.yml b/filebeat/module/auditd/log/_meta/fields.yml
@@ -77,3 +77,7 @@
           type: geo_point
           description: >
             The longitude and latitude.
+        - name: region_iso_code
+          type: keyword
+          description: >
+            Region ISO code.
diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2018-06-27T06:22:36.186Z", 
+        "@timestamp": "2018-07-23T11:50:38.896Z", 
         "fileset.module": "icinga", 
         "fileset.name": "startup", 
         "icinga.startup.facility": "cli", 
@@ -11,7 +11,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-06-27T06:22:36.186Z", 
+        "@timestamp": "2018-07-23T11:50:38.896Z", 
         "fileset.module": "icinga", 
         "fileset.name": "startup", 
         "icinga.startup.facility": "cli", 

diff --git a/filebeat/module/iis/access/_meta/fields.yml b/filebeat/module/iis/access/_meta/fields.yml
@@ -157,3 +157,7 @@
           type: keyword
           description: >
             The city name.
+        - name: region_iso_code
+          type: keyword
+          description: >
+            Region ISO code.
diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json
@@ -8,6 +8,7 @@
         "iis.access.geoip.country_iso_code": "DE", 
         "iis.access.geoip.location.lat": 52.5167, 
         "iis.access.geoip.location.lon": 13.4, 
+        "iis.access.geoip.region_iso_code": "DE-BE", 
         "iis.access.geoip.region_name": "Land Berlin", 
         "iis.access.method": "GET", 
         "iis.access.port": "80", 
@@ -73,6 +74,7 @@
         "iis.access.geoip.country_iso_code": "DE", 
         "iis.access.geoip.location.lat": 52.5167, 
         "iis.access.geoip.location.lon": 13.4, 
+        "iis.access.geoip.region_iso_code": "DE-BE", 
         "iis.access.geoip.region_name": "Land Berlin", 
         "iis.access.hostname": "example.com", 
         "iis.access.http_version": "1.1", 

diff --git a/filebeat/module/iis/error/_meta/fields.yml b/filebeat/module/iis/error/_meta/fields.yml
@@ -71,3 +71,7 @@
           type: keyword
           description: >
             The city name.
+        - name: region_iso_code
+          type: keyword
+          description: >
+            Region ISO code.
diff --git a/filebeat/module/iis/error/test/test.log-expected.json b/filebeat/module/iis/error/test/test.log-expected.json
@@ -26,6 +26,7 @@
         "iis.error.geoip.country_iso_code": "DE", 
         "iis.error.geoip.location.lat": 52.5167, 
         "iis.error.geoip.location.lon": 13.4, 
+        "iis.error.geoip.region_iso_code": "DE-BE", 
         "iis.error.geoip.region_name": "Land Berlin", 
         "iis.error.http_version": "1.1", 
         "iis.error.method": "GET", 
@@ -50,6 +51,7 @@
         "iis.error.geoip.country_iso_code": "DE", 
         "iis.error.geoip.location.lat": 52.5167, 
         "iis.error.geoip.location.lon": 13.4, 
+        "iis.error.geoip.region_iso_code": "DE-BE", 
         "iis.error.geoip.region_name": "Land Berlin", 
         "iis.error.http_version": "2.0", 
         "iis.error.method": "GET", 
@@ -74,6 +76,7 @@
         "iis.error.geoip.country_iso_code": "DE", 
         "iis.error.geoip.location.lat": 52.5167, 
         "iis.error.geoip.location.lon": 13.4, 
+        "iis.error.geoip.region_iso_code": "DE-BE", 
         "iis.error.geoip.region_name": "Land Berlin", 
         "iis.error.queue_name": "-", 
         "iis.error.reason_phrase": "Timer_MinBytesPerSecond", 

diff --git a/filebeat/module/kafka/log/test/state-change-1.1.0.log-expected.json b/filebeat/module/kafka/log/test/state-change-1.1.0.log-expected.json
@@ -1,15 +1,15 @@
 [
     {
-        "@timestamp": "2018-07-16T10:17:06.489Z",
-        "fileset.name": "log",
-        "fileset.module": "kafka",
-        "kafka.log.message": "Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8",
-        "kafka.log.component": "Broker id=30",
-        "kafka.log.class": "state.change.logger",
-        "kafka.log.level": "TRACE",
-        "message": "[2018-07-16 10:17:06,489] TRACE [Broker id=30] Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8 (state.change.logger)",
-        "offset": 0,
-        "input.type": "log",
+        "@timestamp": "2018-07-16T10:17:06.489Z", 
+        "fileset.module": "kafka", 
+        "fileset.name": "log", 
+        "input.type": "log", 
+        "kafka.log.class": "state.change.logger", 
+        "kafka.log.component": "Broker id=30", 
+        "kafka.log.level": "TRACE", 
+        "kafka.log.message": "Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8", 
+        "message": "[2018-07-16 10:17:06,489] TRACE [Broker id=30] Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8 (state.change.logger)", 
+        "offset": 0, 
         "prospector.type": "log"
     }
-]
+]
diff --git a/filebeat/module/nginx/access/_meta/fields.yml b/filebeat/module/nginx/access/_meta/fields.yml
@@ -119,4 +119,7 @@
           type: keyword
           description: >
             The city name.
-
+        - name: region_iso_code
+          type: keyword
+          description: >
+            Region ISO code.
diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json
@@ -65,6 +65,7 @@
         "nginx.access.geoip.country_iso_code": "DE", 
         "nginx.access.geoip.location.lat": 52.5167, 
         "nginx.access.geoip.location.lon": 13.4, 
+        "nginx.access.geoip.region_iso_code": "DE-BE", 
         "nginx.access.geoip.region_name": "Land Berlin", 
         "nginx.access.http_version": "1.1", 
         "nginx.access.method": "GET", 
@@ -100,6 +101,7 @@
         "nginx.access.geoip.country_iso_code": "DE", 
         "nginx.access.geoip.location.lat": 52.5167, 
         "nginx.access.geoip.location.lon": 13.4, 
+        "nginx.access.geoip.region_iso_code": "DE-BE", 
         "nginx.access.geoip.region_name": "Land Berlin", 
         "nginx.access.http_version": "1.1", 
         "nginx.access.method": "GET", 
@@ -133,6 +135,7 @@
         "nginx.access.geoip.country_iso_code": "US", 
         "nginx.access.geoip.location.lat": 39.772, 
         "nginx.access.geoip.location.lon": -89.6859, 
+        "nginx.access.geoip.region_iso_code": "US-IL", 
         "nginx.access.geoip.region_name": "Illinois", 
         "nginx.access.http_version": "1.1", 
         "nginx.access.method": "GET", 

diff --git a/filebeat/module/system/auth/_meta/fields.yml b/filebeat/module/system/auth/_meta/fields.yml
@@ -79,7 +79,10 @@
             type: geo_point
             description: >
               The longitude and latitude.
-
+          - name: region_iso_code
+            type: keyword
+            description: >
+              Region ISO code.
     - name: sudo
       type: group
       description: >

diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json
@@ -61,6 +61,7 @@
         "system.auth.ssh.geoip.country_iso_code": "CN", 
         "system.auth.ssh.geoip.location.lat": 22.5333, 
         "system.auth.ssh.geoip.location.lon": 114.1333, 
+        "system.auth.ssh.geoip.region_iso_code": "CN-44", 
         "system.auth.ssh.geoip.region_name": "Guangdong", 
         "system.auth.ssh.ip": "116.31.116.24", 
         "system.auth.ssh.method": "password", 

diff --git a/filebeat/module/traefik/access/_meta/fields.yml b/filebeat/module/traefik/access/_meta/fields.yml
@@ -112,6 +112,10 @@
           type: keyword
           description: >
             The city name.
+        - name: region_iso_code
+          type: keyword
+          description: >
+            Region ISO code.
     - name: request_count
       type: long
       description: >

diff --git a/filebeat/module/traefik/access/test/test.log-expected.json b/filebeat/module/traefik/access/test/test.log-expected.json
@@ -35,6 +35,7 @@
         "traefik.access.geoip.country_iso_code": "DE", 
         "traefik.access.geoip.location.lat": 52.5167, 
         "traefik.access.geoip.location.lon": 13.4, 
+        "traefik.access.geoip.region_iso_code": "DE-BE", 
         "traefik.access.geoip.region_name": "Land Berlin", 
         "traefik.access.http_version": "1.1", 
         "traefik.access.method": "GET", 

diff --git a/testing/environments/args.yml b/testing/environments/args.yml
@@ -7,4 +7,4 @@ services:
       args:
         DOWNLOAD_URL: https://snapshots.elastic.co/downloads
         ELASTIC_VERSION: 7.0.0-alpha1-SNAPSHOT
-        CACHE_BUST: 20180501
+        CACHE_BUST: 20180723