Promoting integrations packages to snapshot

packages/apache/0.1.0/dataset/access/agent/stream/log.yml.hbs

@@ -0,0 +1,10 @@
+paths:
+{{#each paths as |path i|}}
+ - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

packages/apache/0.1.0/dataset/access/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,101 @@
+---
+description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins."
+
+processors:
+- grok:
+    field: message
+    patterns:
+    - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
+      "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
+      %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
+      "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
+    - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
+      "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
+      %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
+      "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
+    - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
+      "-" %{NUMBER:http.response.status_code:long} -'
+    - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol}
+      %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url.original}
+      HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long})
+    ignore_missing: true
+- remove:
+    field: message
+- set:
+    field: event.kind
+    value: event
+- set:
+    field: event.category
+    value: web
+- set:
+    field: event.outcome
+    value: success
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
+- set:
+    field: event.outcome
+    value: failure
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"
+- grok:
+    field: source.address
+    ignore_missing: true
+    patterns:
+    - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
+- rename:
+    field: '@timestamp'
+    target_field: event.created
+- date:
+    field: apache.access.time
+    target_field: '@timestamp'
+    formats:
+    - dd/MMM/yyyy:H:m:s Z
+    ignore_failure: true
+- remove:
+    field: apache.access.time
+    ignore_failure: true
+- user_agent:
+    field: user_agent.original
+    ignore_failure: true
+- geoip:
+    field: source.ip
+    target_field: source.geo
+    ignore_missing: true
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: source.ip
+    target_field: source.as
+    properties:
+    - asn
+    - organization_name
+    ignore_missing: true
+- rename:
+    field: source.as.asn
+    target_field: source.as.number
+    ignore_missing: true
+- rename:
+    field: source.as.organization_name
+    target_field: source.as.organization.name
+    ignore_missing: true
+- set:
+    field: tls.cipher
+    value: '{{apache.access.ssl.cipher}}'
+    if: ctx?.apache?.access?.ssl?.cipher != null
+
+- script:
+    lang: painless
+    if: ctx?.apache?.access?.ssl?.protocol != null
+    source: >-
+      def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v");
+      if (parts.length != 2) {
+        return;
+      }
+      if (parts[1].contains(".")) {
+        ctx.tls.version = parts[1];
+      } else {
+        ctx.tls.version = parts[1] + ".0";
+      }
+      ctx.tls.version_protocol = parts[0];
+
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

packages/apache/0.1.0/dataset/access/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: dataset.type
+  type: constant_keyword
+  description: Dataset type.
+- name: dataset.name
+  type: constant_keyword
+  description: Dataset name.
+- name: dataset.namespace
+  type: constant_keyword
+  description: Dataset namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/apache/0.1.0/dataset/access/fields/ecs.yml

@@ -0,0 +1,187 @@
+- name: message
+  level: core
+  type: text
+  description: |-
+    For log events the message field contains the log message, optimized for viewing in a log viewer.
+    For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event.
+    If multiple messages exist, they can be combined into one message.
+- name: http
+  title: HTTP
+  group: 2
+  type: group
+  fields:
+  - name: request.method
+    level: extended
+    type: keyword
+    description: |-
+      HTTP request method.
+      Prior to ECS 1.6.0 the following guidance was provided:
+      "The field value must be normalized to lowercase for querying."
+      As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection.  Original case will be mandated in ECS 2.0.0
+    ignore_above: 1024
+  - name: request.referrer
+    level: extended
+    type: keyword
+    description: Referrer for this HTTP request.
+    ignore_above: 1024
+  - name: response.body.bytes
+    level: extended
+    type: long
+    format: bytes
+    description: Size in bytes of the response body.
+  - name: response.status_code
+    level: extended
+    type: long
+    format: string
+    description: HTTP response status code.
+  - name: version
+    level: extended
+    type: keyword
+    description: HTTP version.
+    ignore_above: 1024
+- name: log
+  title: Log
+  group: 2
+  type: group
+  fields:
+  - name: level
+    level: core
+    type: keyword
+    description: |-
+      Original log level of the log event.
+      If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity).
+      Some examples are `warn`, `err`, `i`, `informational`.
+    ignore_above: 1024
+- name: process
+  title: Process
+  group: 2
+  type: group
+  fields:
+  - name: pid
+    level: core
+    type: long
+    format: string
+    description: Process id.
+  - name: thread.id
+    level: extended
+    type: long
+    format: string
+    description: Thread ID.
+- name: source
+  title: Source
+  group: 2
+  type: group
+  fields:
+  - name: address
+    level: extended
+    type: keyword
+    description: |-
+      Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field.
+      Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
+    ignore_above: 1024
+  - name: geo.city_name
+    level: core
+    type: keyword
+    description: City name.
+    ignore_above: 1024
+  - name: geo.continent_name
+    level: core
+    type: keyword
+    description: Name of the continent.
+    ignore_above: 1024
+  - name: geo.country_iso_code
+    level: core
+    type: keyword
+    description: Country ISO code.
+    ignore_above: 1024
+  - name: geo.location
+    level: core
+    type: geo_point
+    description: Longitude and latitude.
+  - name: geo.region_iso_code
+    level: core
+    type: keyword
+    description: Region ISO code.
+    ignore_above: 1024
+  - name: geo.region_name
+    level: core
+    type: keyword
+    description: Region name.
+    ignore_above: 1024
+- name: url
+  title: URL
+  group: 2
+  type: group
+  fields:
+  - name: original
+    level: extended
+    type: keyword
+    description: |-
+      Unmodified original url as seen in the event source.
+      Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path.
+      This field is meant to represent the URL as it was observed, complete or not.
+    ignore_above: 1024
+    multi_fields:
+    - name: text
+      type: text
+      norms: false
+      default_field: false
+- name: user
+  title: User
+  group: 2
+  type: group
+  fields:
+  - name: name
+    level: core
+    type: keyword
+    description: Short name or login of the user.
+    ignore_above: 1024
+    multi_fields:
+    - name: text
+      type: text
+      norms: false
+      default_field: false
+- name: user_agent
+  title: User agent
+  group: 2
+  type: group
+  fields:
+  - name: device.name
+    level: extended
+    type: keyword
+    description: Name of the device.
+    ignore_above: 1024
+  - name: name
+    level: extended
+    type: keyword
+    description: Name of the user agent.
+    ignore_above: 1024
+  - name: original
+    level: extended
+    type: keyword
+    description: Unparsed user_agent string.
+    ignore_above: 1024
+    multi_fields:
+    - name: text
+      type: text
+      norms: false
+  - name: os.name
+    level: extended
+    type: keyword
+    description: Operating system name, without the version.
+    ignore_above: 1024
+    multi_fields:
+    - name: text
+      type: text
+      norms: false
+      default_field: false
+  - name: os.version
+    level: extended
+    type: keyword
+    ignore_above: 1024
+    description: Operating system version as a raw string.
+  - name: version
+    level: extended
+    type: keyword
+    ignore_above: 1024
+    description: Version of the user agent.

packages/apache/0.1.0/dataset/access/fields/fields.yml

@@ -0,0 +1,11 @@
+- name: apache.access
+  type: group
+  fields:
+  - name: ssl.protocol
+    type: keyword
+    description: |
+      SSL protocol version.
+  - name: ssl.cipher
+    type: keyword
+    description: |
+      SSL cipher name.

packages/apache/0.1.0/dataset/access/manifest.yml

@@ -0,0 +1,19 @@
+title: Apache access logs
+release: beta
+type: logs
+streams:
+- input: logfile
+  vars:
+  - name: paths
+    type: text
+    title: Paths
+    multi: true
+    required: true
+    show_user: true
+    default:
+    - /var/log/apache2/access.log*
+    - /var/log/apache2/other_vhosts_access.log*
+    - /var/log/httpd/access_log*
+  template_path: log.yml.hbs
+  title: Apache access logs
+  description: Collect Apache access logs

packages/apache/0.1.0/dataset/error/agent/stream/log.yml.hbs

@@ -0,0 +1,11 @@
+paths:
+{{#each paths as |path i|}}
+ - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0