From b70ce6966c17724304cff0c4912d16586f3d44dd Mon Sep 17 00:00:00 2001 From: Nick Peihl Date: Fri, 20 Jul 2018 11:23:29 -0700 Subject: [PATCH] Add region ISO code to GeoIP Ingest plugin (#31669) --- docs/plugins/ingest-geoip.asciidoc | 4 ++-- .../ingest/geoip/GeoIpProcessor.java | 18 +++++++++++++++--- .../geoip/GeoIpProcessorFactoryTests.java | 2 +- .../ingest/geoip/GeoIpProcessorTests.java | 3 ++- .../test/ingest_geoip/20_geoip_processor.yml | 11 +++++++---- 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/docs/plugins/ingest-geoip.asciidoc b/docs/plugins/ingest-geoip.asciidoc index 32516d07bef37..688b36042ea59 100644 --- a/docs/plugins/ingest-geoip.asciidoc +++ b/docs/plugins/ingest-geoip.asciidoc @@ -26,14 +26,14 @@ include::install_remove.asciidoc[] | `field` | yes | - | The field to get the ip address from for the geographical lookup. | `target_field` | no | geoip | The field that will hold the geographical information looked up from the Maxmind database. | `database_file` | no | GeoLite2-City.mmdb | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb, GeoLite2-Country.mmdb and GeoLite2-ASN.mmdb files. -| `properties` | no | [`continent_name`, `country_iso_code`, `region_name`, `city_name`, `location`] * | Controls what properties are added to the `target_field` based on the geoip lookup. +| `properties` | no | [`continent_name`, `country_iso_code`, `region_iso_code`, `region_name`, `city_name`, `location`] * | Controls what properties are added to the `target_field` based on the geoip lookup. | `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document |====== *Depends on what is available in `database_field`: * If the GeoLite2 City database is used, then the following fields may be added under the `target_field`: `ip`, -`country_iso_code`, `country_name`, `continent_name`, `region_name`, `city_name`, `timezone`, `latitude`, `longitude` +`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `latitude`, `longitude` and `location`. The fields actually added depend on what has been found and which properties were configured in `properties`. * If the GeoLite2 Country database is used, then the following fields may be added under the `target_field`: `ip`, `country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index f1b4b33017e3d..366b6ffc1d241 100644 --- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -185,6 +185,16 @@ private Map retrieveCityGeoData(InetAddress ipAddress) { geoData.put("continent_name", continentName); } break; + case REGION_ISO_CODE: + // ISO 3166-2 code for country subdivisions. + // See iso.org/iso-3166-country-codes.html + String countryIso = country.getIsoCode(); + String subdivisionIso = subdivision.getIsoCode(); + if (countryIso != null && subdivisionIso != null) { + String regionIsoCode = countryIso + "-" + subdivisionIso; + geoData.put("region_iso_code", regionIsoCode); + } + break; case REGION_NAME: String subdivisionName = subdivision.getName(); if (subdivisionName != null) { @@ -300,8 +310,8 @@ private Map retrieveAsnGeoData(InetAddress ipAddress) { public static final class Factory implements Processor.Factory { static final Set DEFAULT_CITY_PROPERTIES = EnumSet.of( - Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_NAME, - Property.CITY_NAME, Property.LOCATION + Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_ISO_CODE, + Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION ); static final Set DEFAULT_COUNTRY_PROPERTIES = EnumSet.of( Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE @@ -377,6 +387,7 @@ enum Property { COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME, + REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, @@ -386,7 +397,8 @@ enum Property { static final EnumSet ALL_CITY_PROPERTIES = EnumSet.of( Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME, - Property.REGION_NAME, Property.CITY_NAME, Property.TIMEZONE, Property.LOCATION + Property.REGION_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.TIMEZONE, + Property.LOCATION ); static final EnumSet ALL_COUNTRY_PROPERTIES = EnumSet.of( Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index 0aa2eb9fdfa3b..7a5d6f5808f76 100644 --- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -284,7 +284,7 @@ public void testBuildIllegalFieldOption() throws Exception { config1.put("properties", Collections.singletonList("invalid")); Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config1)); assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " + - "COUNTRY_NAME, CONTINENT_NAME, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]")); + "COUNTRY_NAME, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]")); Map config2 = new HashMap<>(); config2.put("field", "_field"); diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 48a1769cbf82f..4c04d4e340a71 100644 --- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -117,11 +117,12 @@ public void testCity_withIpV6() throws Exception { assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(address)); @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(8)); + assertThat(geoData.size(), equalTo(9)); assertThat(geoData.get("ip"), equalTo(address)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); assertThat(geoData.get("continent_name"), equalTo("North America")); + assertThat(geoData.get("region_iso_code"), equalTo("US-FL")); assertThat(geoData.get("region_name"), equalTo("Florida")); assertThat(geoData.get("city_name"), equalTo("Hollywood")); assertThat(geoData.get("timezone"), equalTo("America/New_York")); diff --git a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml index 0c400c3c0eabe..012ca7173187c 100644 --- a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml +++ b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml @@ -30,11 +30,12 @@ type: test id: 1 - match: { _source.field1: "128.101.101.101" } - - length: { _source.geoip: 5 } + - length: { _source.geoip: 6 } - match: { _source.geoip.city_name: "Minneapolis" } - match: { _source.geoip.country_iso_code: "US" } - match: { _source.geoip.location.lon: -93.2166 } - match: { _source.geoip.location.lat: 44.9759 } + - match: { _source.geoip.region_iso_code: "US-MN" } - match: { _source.geoip.region_name: "Minnesota" } - match: { _source.geoip.continent_name: "North America" } @@ -54,7 +55,7 @@ { "geoip" : { "field" : "field1", - "properties" : ["city_name", "country_iso_code", "ip", "location", "timezone", "country_name", "region_name", "continent_name"] + "properties" : ["city_name", "country_iso_code", "ip", "location", "timezone", "country_name", "region_iso_code", "region_name", "continent_name"] } } ] @@ -75,7 +76,7 @@ type: test id: 1 - match: { _source.field1: "128.101.101.101" } - - length: { _source.geoip: 8 } + - length: { _source.geoip: 9 } - match: { _source.geoip.city_name: "Minneapolis" } - match: { _source.geoip.country_iso_code: "US" } - match: { _source.geoip.ip: "128.101.101.101" } @@ -83,6 +84,7 @@ - match: { _source.geoip.location.lat: 44.9759 } - match: { _source.geoip.timezone: "America/Chicago" } - match: { _source.geoip.country_name: "United States" } + - match: { _source.geoip.region_iso_code: "US-MN" } - match: { _source.geoip.region_name: "Minnesota" } - match: { _source.geoip.continent_name: "North America" } @@ -188,11 +190,12 @@ type: test id: 2 - match: { _source.field1: "128.101.101.101" } - - length: { _source.geoip: 5 } + - length: { _source.geoip: 6 } - match: { _source.geoip.city_name: "Minneapolis" } - match: { _source.geoip.country_iso_code: "US" } - match: { _source.geoip.location.lon: -93.2166 } - match: { _source.geoip.location.lat: 44.9759 } + - match: { _source.geoip.region_iso_code: "US-MN" } - match: { _source.geoip.region_name: "Minnesota" } - match: { _source.geoip.continent_name: "North America" }