From 84815e4a7316f67f1630b0b68524385ff83958c8 Mon Sep 17 00:00:00 2001 From: Agi K Thomas <101976829+agithomas@users.noreply.github.com> Date: Thu, 16 Mar 2023 16:36:33 +0530 Subject: [PATCH] Nginx TSDB Enablement (#5315) * Nginx TSDB Enablement * Added inline comment on dimension fields --- packages/nginx/changelog.yml | 5 + .../data_stream/stubstatus/fields/agent.yml | 13 +++ .../data_stream/stubstatus/fields/ecs.yml | 1 + .../data_stream/stubstatus/fields/fields.yml | 11 ++ packages/nginx/docs/README.md | 103 +++++++++--------- ...-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json | 4 +- ...-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json | 6 +- ...-d763a570-f1a1-11e7-a9ef-93c69af7b129.json | 4 +- packages/nginx/manifest.yml | 2 +- 9 files changed, 90 insertions(+), 59 deletions(-) diff --git a/packages/nginx/changelog.yml b/packages/nginx/changelog.yml index 729d213b395..f4c0a3ff2d2 100644 --- a/packages/nginx/changelog.yml +++ b/packages/nginx/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.8.0" + changes: + - description: Added changes to support TSDB Enablement. + type: enhancement + link: https://github.com/elastic/integrations/pull/5315 - version: "1.7.1" changes: - description: Added categories and/or subcategories. diff --git a/packages/nginx/data_stream/stubstatus/fields/agent.yml b/packages/nginx/data_stream/stubstatus/fields/agent.yml index da4e652c53b..fedf28e5ffd 100644 --- a/packages/nginx/data_stream/stubstatus/fields/agent.yml +++ b/packages/nginx/data_stream/stubstatus/fields/agent.yml @@ -25,6 +25,7 @@ ignore_above: 1024 description: Instance ID of the host machine. example: i-1234567890abcdef0 + dimension: true - name: instance.name level: extended type: keyword @@ -42,6 +43,7 @@ ignore_above: 1024 description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. example: aws + dimension: true - name: region level: extended type: keyword @@ -51,6 +53,7 @@ - name: project.id type: keyword description: Name of the project in Google Cloud. + dimension: true - name: image.id type: keyword description: Image ID for the cloud instance. @@ -67,6 +70,7 @@ type: keyword ignore_above: 1024 description: Unique container id. + dimension: true - name: image.name level: extended type: keyword @@ -134,6 +138,7 @@ level: core type: keyword ignore_above: 1024 + dimension: true description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' @@ -196,3 +201,11 @@ description: > OS codename, if any. +- name: agent + title: Agent + type: group + fields: + - name: id + type: keyword + ignore_above: 1024 + dimension: true \ No newline at end of file diff --git a/packages/nginx/data_stream/stubstatus/fields/ecs.yml b/packages/nginx/data_stream/stubstatus/fields/ecs.yml index aae79b4b9fc..d7ea22de8d6 100644 --- a/packages/nginx/data_stream/stubstatus/fields/ecs.yml +++ b/packages/nginx/data_stream/stubstatus/fields/ecs.yml @@ -4,5 +4,6 @@ name: ecs.version - external: ecs name: service.address + dimension: true - external: ecs name: service.type diff --git a/packages/nginx/data_stream/stubstatus/fields/fields.yml b/packages/nginx/data_stream/stubstatus/fields/fields.yml index 5ded312df6f..f77ea6f82b2 100644 --- a/packages/nginx/data_stream/stubstatus/fields/fields.yml +++ b/packages/nginx/data_stream/stubstatus/fields/fields.yml @@ -3,41 +3,52 @@ fields: - name: hostname type: keyword + # Reason to add as a dimension field: This can be cluster url or single host. This is a good to have dimension. + dimension: true description: | Nginx hostname. - name: active type: long + metric_type: gauge description: | The current number of active client connections including Waiting connections. - name: accepts type: long + metric_type: counter description: | The total number of accepted client connections. - name: handled type: long + metric_type: counter description: | The total number of handled client connections. - name: dropped type: long + metric_type: counter description: | The total number of dropped client connections. - name: requests type: long + metric_type: counter description: | The total number of client requests. - name: current type: long + metric_type: gauge description: | The current number of client requests. - name: reading type: long + metric_type: gauge description: | The current number of connections where Nginx is reading the request header. - name: writing type: long + metric_type: gauge description: | The current number of connections where Nginx is writing the response back to the client. - name: waiting type: long + metric_type: gauge description: | The current number of idle client connections waiting for a request. diff --git a/packages/nginx/docs/README.md b/packages/nginx/docs/README.md index cf400226cbb..672ec29f2bf 100644 --- a/packages/nginx/docs/README.md +++ b/packages/nginx/docs/README.md @@ -516,57 +516,58 @@ An example event for `stubstatus` looks as following: **Exported fields** -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.dataset | Event dataset | constant_keyword | -| event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| nginx.stubstatus.accepts | The total number of accepted client connections. | long | -| nginx.stubstatus.active | The current number of active client connections including Waiting connections. | long | -| nginx.stubstatus.current | The current number of client requests. | long | -| nginx.stubstatus.dropped | The total number of dropped client connections. | long | -| nginx.stubstatus.handled | The total number of handled client connections. | long | -| nginx.stubstatus.hostname | Nginx hostname. | keyword | -| nginx.stubstatus.reading | The current number of connections where Nginx is reading the request header. | long | -| nginx.stubstatus.requests | The total number of client requests. | long | -| nginx.stubstatus.waiting | The current number of idle client connections waiting for a request. | long | -| nginx.stubstatus.writing | The current number of connections where Nginx is writing the response back to the client. | long | -| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | +| Field | Description | Type | Metric Type | +|---|---|---|---| +| @timestamp | Event timestamp. | date | | +| agent.id | | keyword | | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | | +| cloud.image.id | Image ID for the cloud instance. | keyword | | +| cloud.instance.id | Instance ID of the host machine. | keyword | | +| cloud.instance.name | Instance name of the host machine. | keyword | | +| cloud.machine.type | Machine type of the host machine. | keyword | | +| cloud.project.id | Name of the project in Google Cloud. | keyword | | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | +| cloud.region | Region in which this host is running. | keyword | | +| container.id | Unique container id. | keyword | | +| container.image.name | Name of the image the container was built on. | keyword | | +| container.labels | Image labels. | object | | +| container.name | Container name. | keyword | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | +| data_stream.type | Data stream type. | constant_keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | +| event.dataset | Event dataset | constant_keyword | | +| event.module | Event module | constant_keyword | | +| host.architecture | Operating system architecture. | keyword | | +| host.containerized | If the host is a container. | boolean | | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | +| host.ip | Host ip addresses. | ip | | +| host.mac | Host mac addresses. | keyword | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| host.os.build | OS build information. | keyword | | +| host.os.codename | OS codename, if any. | keyword | | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | | +| host.os.name | Operating system name, without the version. | keyword | | +| host.os.name.text | Multi-field of `host.os.name`. | text | | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | +| host.os.version | Operating system version as a raw string. | keyword | | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | +| nginx.stubstatus.accepts | The total number of accepted client connections. | long | counter | +| nginx.stubstatus.active | The current number of active client connections including Waiting connections. | long | gauge | +| nginx.stubstatus.current | The current number of client requests. | long | gauge | +| nginx.stubstatus.dropped | The total number of dropped client connections. | long | counter | +| nginx.stubstatus.handled | The total number of handled client connections. | long | counter | +| nginx.stubstatus.hostname | Nginx hostname. | keyword | | +| nginx.stubstatus.reading | The current number of connections where Nginx is reading the request header. | long | gauge | +| nginx.stubstatus.requests | The total number of client requests. | long | counter | +| nginx.stubstatus.waiting | The current number of idle client connections waiting for a request. | long | gauge | +| nginx.stubstatus.writing | The current number of connections where Nginx is writing the response back to the client. | long | gauge | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | ## ML Modules diff --git a/packages/nginx/kibana/visualization/nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json b/packages/nginx/kibana/visualization/nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json index c0e4582c21a..7aebbafe6bd 100644 --- a/packages/nginx/kibana/visualization/nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json +++ b/packages/nginx/kibana/visualization/nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json @@ -40,7 +40,7 @@ { "field": "nginx.stubstatus.requests", "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" + "type": "max" }, { "field": "61ca57f2-469d-11e7-af02-69e470af7417", @@ -72,4 +72,4 @@ }, "references": [], "type": "visualization" -} \ No newline at end of file +} diff --git a/packages/nginx/kibana/visualization/nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json b/packages/nginx/kibana/visualization/nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json index 2bb31a5e483..6b3341217b2 100644 --- a/packages/nginx/kibana/visualization/nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json +++ b/packages/nginx/kibana/visualization/nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json @@ -51,7 +51,7 @@ { "field": "nginx.stubstatus.accepts", "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" + "type": "max" }, { "field": "61ca57f2-469d-11e7-af02-69e470af7417", @@ -79,7 +79,7 @@ { "field": "nginx.stubstatus.handled", "id": "56dd33b1-f1a3-11e7-95d0-8ddf041d42a2", - "type": "avg" + "type": "max" }, { "field": "56dd33b1-f1a3-11e7-95d0-8ddf041d42a2", @@ -111,4 +111,4 @@ }, "references": [], "type": "visualization" -} \ No newline at end of file +} diff --git a/packages/nginx/kibana/visualization/nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129.json b/packages/nginx/kibana/visualization/nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129.json index 0f8d58d94a6..4c1ae42f66d 100644 --- a/packages/nginx/kibana/visualization/nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129.json +++ b/packages/nginx/kibana/visualization/nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129.json @@ -40,7 +40,7 @@ { "field": "nginx.stubstatus.dropped", "id": "61ca57f2-469d-11e7-af02-69e470af7417", - "type": "avg" + "type": "max" }, { "field": "61ca57f2-469d-11e7-af02-69e470af7417", @@ -72,4 +72,4 @@ }, "references": [], "type": "visualization" -} \ No newline at end of file +} diff --git a/packages/nginx/manifest.yml b/packages/nginx/manifest.yml index d3aae8b69d0..964bc75eb75 100644 --- a/packages/nginx/manifest.yml +++ b/packages/nginx/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: nginx title: Nginx -version: 1.7.1 +version: 1.8.0 license: basic description: Collect logs and metrics from Nginx HTTP servers with Elastic Agent. type: integration