Nginx TSDB Enablement (#5315)

* Nginx TSDB Enablement * Added inline comment on dimension fields
elastic · Mar 16, 2023 · 84815e4 · 84815e4
1 parent 78da950
commit 84815e4
Show file tree

Hide file tree

Showing 9 changed files with 90 additions and 59 deletions.
diff --git a/packages/nginx/changelog.yml b/packages/nginx/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.8.0"
+  changes:
+    - description: Added changes to support TSDB Enablement.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5315
 - version: "1.7.1"
   changes:
     - description: Added categories and/or subcategories.

diff --git a/packages/nginx/data_stream/stubstatus/fields/agent.yml b/packages/nginx/data_stream/stubstatus/fields/agent.yml
@@ -25,6 +25,7 @@
       ignore_above: 1024
       description: Instance ID of the host machine.
       example: i-1234567890abcdef0
+      dimension: true
     - name: instance.name
       level: extended
       type: keyword
@@ -42,6 +43,7 @@
       ignore_above: 1024
       description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
       example: aws
+      dimension: true
     - name: region
       level: extended
       type: keyword
@@ -51,6 +53,7 @@
     - name: project.id
       type: keyword
       description: Name of the project in Google Cloud.
+      dimension: true
     - name: image.id
       type: keyword
       description: Image ID for the cloud instance.
@@ -67,6 +70,7 @@
       type: keyword
       ignore_above: 1024
       description: Unique container id.
+      dimension: true
     - name: image.name
       level: extended
       type: keyword
@@ -134,6 +138,7 @@
       level: core
       type: keyword
       ignore_above: 1024
+      dimension: true
       description: 'Name of the host.
 
         It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'
@@ -196,3 +201,11 @@
       description: >
         OS codename, if any.
 
+- name: agent
+  title: Agent
+  type: group
+  fields:
+    - name: id
+      type: keyword
+      ignore_above: 1024
+      dimension: true
diff --git a/packages/nginx/data_stream/stubstatus/fields/ecs.yml b/packages/nginx/data_stream/stubstatus/fields/ecs.yml
@@ -4,5 +4,6 @@
   name: ecs.version
 - external: ecs
   name: service.address
+  dimension: true
 - external: ecs
   name: service.type
diff --git a/packages/nginx/data_stream/stubstatus/fields/fields.yml b/packages/nginx/data_stream/stubstatus/fields/fields.yml
@@ -3,41 +3,52 @@
   fields:
     - name: hostname
       type: keyword
+      # Reason to add as a dimension field: This can be cluster url or single host. This is a good to have dimension.
+      dimension: true
       description: |
         Nginx hostname.
     - name: active
       type: long
+      metric_type: gauge
       description: |
         The current number of active client connections including Waiting connections.
     - name: accepts
       type: long
+      metric_type: counter
       description: |
         The total number of accepted client connections.
     - name: handled
       type: long
+      metric_type: counter
       description: |
         The total number of handled client connections.
     - name: dropped
       type: long
+      metric_type: counter
       description: |
         The total number of dropped client connections.
     - name: requests
       type: long
+      metric_type: counter
       description: |
         The total number of client requests.
     - name: current
       type: long
+      metric_type: gauge
       description: |
         The current number of client requests.
     - name: reading
       type: long
+      metric_type: gauge
       description: |
         The current number of connections where Nginx is reading the request header.
     - name: writing
       type: long
+      metric_type: gauge
       description: |
         The current number of connections where Nginx is writing the response back to the client.
     - name: waiting
       type: long
+      metric_type: gauge
       description: |
         The current number of idle client connections waiting for a request.
diff --git a/packages/nginx/docs/README.md b/packages/nginx/docs/README.md
@@ -516,57 +516,58 @@ An example event for `stubstatus` looks as following:
 
 **Exported fields**
 
-| Field | Description | Type |
-|---|---|---|
-| @timestamp | Event timestamp. | date |
-| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |
-| cloud.availability_zone | Availability zone in which this host is running. | keyword |
-| cloud.image.id | Image ID for the cloud instance. | keyword |
-| cloud.instance.id | Instance ID of the host machine. | keyword |
-| cloud.instance.name | Instance name of the host machine. | keyword |
-| cloud.machine.type | Machine type of the host machine. | keyword |
-| cloud.project.id | Name of the project in Google Cloud. | keyword |
-| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword |
-| cloud.region | Region in which this host is running. | keyword |
-| container.id | Unique container id. | keyword |
-| container.image.name | Name of the image the container was built on. | keyword |
-| container.labels | Image labels. | object |
-| container.name | Container name. | keyword |
-| data_stream.dataset | Data stream dataset. | constant_keyword |
-| data_stream.namespace | Data stream namespace. | constant_keyword |
-| data_stream.type | Data stream type. | constant_keyword |
-| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
-| event.dataset | Event dataset | constant_keyword |
-| event.module | Event module | constant_keyword |
-| host.architecture | Operating system architecture. | keyword |
-| host.containerized | If the host is a container. | boolean |
-| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword |
-| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword |
-| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |
-| host.ip | Host ip addresses. | ip |
-| host.mac | Host mac addresses. | keyword |
-| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |
-| host.os.build | OS build information. | keyword |
-| host.os.codename | OS codename, if any. | keyword |
-| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |
-| host.os.kernel | Operating system kernel version as a raw string. | keyword |
-| host.os.name | Operating system name, without the version. | keyword |
-| host.os.name.text | Multi-field of `host.os.name`. | text |
-| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |
-| host.os.version | Operating system version as a raw string. | keyword |
-| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
-| nginx.stubstatus.accepts | The total number of accepted client connections. | long |
-| nginx.stubstatus.active | The current number of active client connections including Waiting connections. | long |
-| nginx.stubstatus.current | The current number of client requests. | long |
-| nginx.stubstatus.dropped | The total number of dropped client connections. | long |
-| nginx.stubstatus.handled | The total number of handled client connections. | long |
-| nginx.stubstatus.hostname | Nginx hostname. | keyword |
-| nginx.stubstatus.reading | The current number of connections where Nginx is reading the request header. | long |
-| nginx.stubstatus.requests | The total number of client requests. | long |
-| nginx.stubstatus.waiting | The current number of idle client connections waiting for a request. | long |
-| nginx.stubstatus.writing | The current number of connections where Nginx is writing the response back to the client. | long |
-| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |
-| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |
+| Field | Description | Type | Metric Type |
+|---|---|---|---|
+| @timestamp | Event timestamp. | date |  |
+| agent.id |  | keyword |  |
+| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |  |
+| cloud.availability_zone | Availability zone in which this host is running. | keyword |  |
+| cloud.image.id | Image ID for the cloud instance. | keyword |  |
+| cloud.instance.id | Instance ID of the host machine. | keyword |  |
+| cloud.instance.name | Instance name of the host machine. | keyword |  |
+| cloud.machine.type | Machine type of the host machine. | keyword |  |
+| cloud.project.id | Name of the project in Google Cloud. | keyword |  |
+| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword |  |
+| cloud.region | Region in which this host is running. | keyword |  |
+| container.id | Unique container id. | keyword |  |
+| container.image.name | Name of the image the container was built on. | keyword |  |
+| container.labels | Image labels. | object |  |
+| container.name | Container name. | keyword |  |
+| data_stream.dataset | Data stream dataset. | constant_keyword |  |
+| data_stream.namespace | Data stream namespace. | constant_keyword |  |
+| data_stream.type | Data stream type. | constant_keyword |  |
+| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |  |
+| event.dataset | Event dataset | constant_keyword |  |
+| event.module | Event module | constant_keyword |  |
+| host.architecture | Operating system architecture. | keyword |  |
+| host.containerized | If the host is a container. | boolean |  |
+| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword |  |
+| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword |  |
+| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |  |
+| host.ip | Host ip addresses. | ip |  |
+| host.mac | Host mac addresses. | keyword |  |
+| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |  |
+| host.os.build | OS build information. | keyword |  |
+| host.os.codename | OS codename, if any. | keyword |  |
+| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |  |
+| host.os.kernel | Operating system kernel version as a raw string. | keyword |  |
+| host.os.name | Operating system name, without the version. | keyword |  |
+| host.os.name.text | Multi-field of `host.os.name`. | text |  |
+| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |  |
+| host.os.version | Operating system version as a raw string. | keyword |  |
+| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |  |
+| nginx.stubstatus.accepts | The total number of accepted client connections. | long | counter |
+| nginx.stubstatus.active | The current number of active client connections including Waiting connections. | long | gauge |
+| nginx.stubstatus.current | The current number of client requests. | long | gauge |
+| nginx.stubstatus.dropped | The total number of dropped client connections. | long | counter |
+| nginx.stubstatus.handled | The total number of handled client connections. | long | counter |
+| nginx.stubstatus.hostname | Nginx hostname. | keyword |  |
+| nginx.stubstatus.reading | The current number of connections where Nginx is reading the request header. | long | gauge |
+| nginx.stubstatus.requests | The total number of client requests. | long | counter |
+| nginx.stubstatus.waiting | The current number of idle client connections waiting for a request. | long | gauge |
+| nginx.stubstatus.writing | The current number of connections where Nginx is writing the response back to the client. | long | gauge |
+| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |  |
+| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |  |
 
 
 ## ML Modules

diff --git a/packages/nginx/kibana/visualization/nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json b/packages/nginx/kibana/visualization/nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json
@@ -40,7 +40,7 @@
                             {
                                 "field": "nginx.stubstatus.requests",
                                 "id": "61ca57f2-469d-11e7-af02-69e470af7417",
-                                "type": "avg"
+                                "type": "max"
                             },
                             {
                                 "field": "61ca57f2-469d-11e7-af02-69e470af7417",
@@ -72,4 +72,4 @@
     },
     "references": [],
     "type": "visualization"
-}
+}
diff --git a/packages/nginx/kibana/visualization/nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json b/packages/nginx/kibana/visualization/nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json
@@ -51,7 +51,7 @@
                             {
                                 "field": "nginx.stubstatus.accepts",
                                 "id": "61ca57f2-469d-11e7-af02-69e470af7417",
-                                "type": "avg"
+                                "type": "max"
                             },
                             {
                                 "field": "61ca57f2-469d-11e7-af02-69e470af7417",
@@ -79,7 +79,7 @@
                             {
                                 "field": "nginx.stubstatus.handled",
                                 "id": "56dd33b1-f1a3-11e7-95d0-8ddf041d42a2",
-                                "type": "avg"
+                                "type": "max"
                             },
                             {
                                 "field": "56dd33b1-f1a3-11e7-95d0-8ddf041d42a2",
@@ -111,4 +111,4 @@
     },
     "references": [],
     "type": "visualization"
-}
+}
diff --git a/packages/nginx/kibana/visualization/nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129.json b/packages/nginx/kibana/visualization/nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129.json
@@ -40,7 +40,7 @@
                             {
                                 "field": "nginx.stubstatus.dropped",
                                 "id": "61ca57f2-469d-11e7-af02-69e470af7417",
-                                "type": "avg"
+                                "type": "max"
                             },
                             {
                                 "field": "61ca57f2-469d-11e7-af02-69e470af7417",
@@ -72,4 +72,4 @@
     },
     "references": [],
     "type": "visualization"
-}
+}
diff --git a/packages/nginx/manifest.yml b/packages/nginx/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: nginx
 title: Nginx
-version: 1.7.1
+version: 1.8.0
 license: basic
 description: Collect logs and metrics from Nginx HTTP servers with Elastic Agent.
 type: integration