Convert apache2.access to ECS - Take 2 (#9245)

- Convert many fields under `apache2.access.*` to ECS. Previous field names are field aliases towards the new corresponding ECS field:
  - apache2.access.user_name => user.name
  - apache2.access.method => http.request.method
  - apache2.access.url => url.original
  - apache2.access.http_version => http.version
  - apache2.access.response_code => http.response.status_code
  - apache2.access.referrer => http.request.referrer
  - apache2.access.agent => user_agent.original
  - read_timestamp => event.created
  - apache2.access.geoip.* => source.geo.*
  - apache2.access.user_agent.* => user_agent.*
- Rename common field `url.href` to `url.original`, bringing it in line with ECS
- Remove field defs which are no longer in ECS:
  - url.href.raw
  - source.hostname
- Add a log entry with a hostname as the remote host, instead of an IP
- Add leniency to geoip filter, if remote host is a hostname instead of an IP (no ip field)
- Coerce int fields to ints: status_code, sent bytes
- Fix unrelated test failure caused by generated code (under x-pack/metricbeat) that seems to be outdated in master

CHANGELOG.asciidoc

@@ -137,6 +137,7 @@ https://github.com/elastic/beats/compare/v6.5.0...v7.0.0-alpha1[View commits]
 - Rename many `haproxy.*` fields to map to ECS. {pull}9117[9117]
 - Rename many `nginx.access.*` fields to map to ECS. {pull}9081[9081]
 - Rename many `system.auth.*` fields to map to ECS. {pull}9138[9138]
+- Rename many `apache2.access.*` fields to map to ECS. {pull}9245[9245]
 
 *Metricbeat*
 

auditbeat/docs/fields.asciidoc

@@ -4387,10 +4387,10 @@ URL fields provide a complete URL, with scheme, host, and path. The URL object c
 
 
 
-*`url.href`*::
+*`url.original`*::
 +
 --
-type: text
+type: keyword
 
 example: https://elastic.co:443/search?q=elasticsearch#top
 
@@ -4399,16 +4399,6 @@ Full url. The field is stored as keyword.
 `href` is an analyzed field so the parsed information can be accessed through `href.analyzed` in queries.
 
 
-*`url.href.raw`*::
-+
---
-type: keyword
-
-The full URL. This is a non-analyzed field that is useful for aggregations.
-
-
---
-
 --
 
 *`url.scheme`*::

dev-tools/ecs-migration.yml

@@ -45,6 +45,10 @@
   alias: true
   copy_to: false
 
+# Filebeat modules
+
+## Suricata module
+
 - from: source_ecs.ip
   to: source.ip
   alias: true
@@ -155,6 +159,59 @@
   alias: true
   copy_to: false
 
+## Apache
+
+- from: apache2.access.user_name
+  to: user.name
+  alias: true
+  copy_to: false
+
+- from: apache2.access.method
+  to: http.request.method
+  alias: true
+  copy_to: false
+
+- from: apache2.access.url
+  to: url.original
+  alias: true
+  copy_to: false
+
+- from: apache2.access.http_version
+  to: http.version
+  alias: true
+  copy_to: false
+
+- from: apache2.access.response_code
+  to: http.response.status_code
+  alias: true
+  copy_to: false
+
+- from: apache2.access.referrer
+  to: http.request.referrer
+  alias: true
+  copy_to: false
+
+- from: apache2.access.agent
+  to: user_agent.original
+  alias: true
+  copy_to: false
+
+- from: read_timestamp
+  to: event.created
+  alias: false
+  copy_to: false
+
+# These expand all fields under geoip and user_agent
+- from: apache2.access.geoip.*
+  to: source.geo.*
+  alias: false
+  copy_to: false
+
+- from: apache2.access.user_agent.*
+  to: user_agent.*
+  alias: false
+  copy_to: false
+
 # From Auditbeat's auditd module.
 - from: source.hostname
   to: source.domain

filebeat/_meta/fields.common.yml

@@ -108,13 +108,6 @@
         Referrer for this HTTP request.
       example: https://blog.example.com/
 
-  # Temporary fixes until ECS is reimported
-    - name: url.original
-      type: keyword
-      description: >
-        Full original url. The field is stored as keyword.
-      example: https://blog.example.com/
-
   # Temporary fix to get 7.0 dashboards working
     - name: fileset.name
       type: alias