From f4fef58757fa566bc1c7c862a043e747c8f0a52d Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Wed, 14 Feb 2024 00:57:27 +0000 Subject: [PATCH 01/19] backport of commit 8fbe164ce9ad7c4a127276472b24494b3453760c --- .../k8s/connect/observability/datadog.mdx | 100 ++++++++++++++++++ website/data/docs-nav-data.json | 4 + 2 files changed, 104 insertions(+) create mode 100644 website/content/docs/k8s/connect/observability/datadog.mdx diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx new file mode 100644 index 000000000000..896a4bdc7c50 --- /dev/null +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -0,0 +1,100 @@ +--- +layout: docs +page_title: Configure Datadog Metrics Collection for Consul on Kubernetes +description: >- + Enable Consul Monitoring via Datadog using the `metrics.datadog` helm annotation. +--- + +# Datadog Integration for Consul on Kubernetes + +The Helm chart includes automated configuration options in order to integrate with Datadog. + +### Note + + + The **DogStatsD**, **Openmetrics Prometheus**, and **Datadog Integration** methods are mutually exclusive and cannot be enabled at the same time. This is currently unsupported within current Datadog Integration. One of the three methods of integration must be enabled at any given time. + + +## Consul Datadog Agent Metrics Integration Checks: + + + + ```yaml + metrics: + enabled: true + enableAgentMetrics: true + datadog: + enabled: true + namespace: "datadog" + ``` + + + + | Consul Component | Description | API Endpoint(s) | + |------------------|--------------------------------------------------|----------------------------------------| + | Serf | Events and Membership Flaps | `/v1/agent/metrics` (prometheus) | + | Raft | Monitors Raft peer information | `/v1/status/leader` `/v1/status/peers` | + | Catalog Services | Service Health Status and Node Count | | + | Catalog Nodes | Node Service Count and Health Status | | + | Consul Latency | Consul LAN + WAN Coordinate Latency Calculations | | + + +## Consul Server Agent DogstatsD Metrics Collection + + + + + ```yaml + metrics: + enabled: true + enableAgentMetrics: true + datadog: + enabled: true + namespace: "datadog" + dogstatsd: + enabled: true + socketTransportType: "UDS" + dogstatsdAddr: "/var/run/datadog/dsd.socket" + ``` + + + + + + ```yaml + metrics: + enabled: true + enableAgentMetrics: true + datadog: + enabled: true + namespace: "datadog" + dogstatsd: + enabled: true + socketTransportType: "UDP" + dogstatsdAddr: "datadog.datadog.svc.cluster.local" + ``` + + + + + - Collects DogstatsD formatted metrics pertaining to Consul Serf Membership, Raft, DNS Performance, Agent Telemetry, and much more. + - Enables configuring DogstatsD metric collection using one of either `UDP` or `Unix Domain Socket configuration` + +## Consul Server Openmetrics Prometheus Metrics Collection + + + + ```yaml + metrics: + enabled: true + enableAgentMetrics: true + datadog: + enabled: true + namespace: "datadog" + openMetricsPrometheus: + enabled: true + ``` + + + + - Collects Consul related metrics via the `/v1/agent/metrics` API endpoint leveraging the `prometheus` query parameter. diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index d2d1b9ab54e5..ce6653e6ead8 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -1408,6 +1408,10 @@ { "title": "Metrics", "path": "k8s/connect/observability/metrics" + }, + { + "title": "Datadog", + "path": "k8s/connect/observability/datadog" } ] } From 04ed4da2af0eea7e2d94eec42f5c215a4b8f4b6a Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Wed, 14 Feb 2024 04:29:57 +0000 Subject: [PATCH 02/19] backport of commit a451d3e338ecc70616b4e0822944fde29169bc4d --- .../k8s/connect/observability/datadog.mdx | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 896a4bdc7c50..86a1a48290ef 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -9,13 +9,18 @@ description: >- The Helm chart includes automated configuration options in order to integrate with Datadog. -### Note +### Datadog Metrics Integration Methods - - The **DogStatsD**, **Openmetrics Prometheus**, and **Datadog Integration** methods are mutually exclusive and cannot be enabled at the same time. This is currently unsupported within current Datadog Integration. One of the three methods of integration must be enabled at any given time. - + + Choose one integration method from the three described below that best suites the intent for metrics collection. Consul Integration, DogStatsD, and Openmetrics Prometheus methods of integration are mutually exclusive. +

+ Reasoning: The consul-k8s helm chart automated configuration implements Datadog's Consul Integration method using the use_prometheus_endpoint configuration parameter. DogstatsD, Consul Integration, and Openmetrics Prometheus Metrics by design share the same metric name syntax for collection, and would therefore cause a conflict. + The consul.py integration source code, as well as the consul-k8s helm chart prohibit the enablement of more that one integration at a time. + -## Consul Datadog Agent Metrics Integration Checks: + + +## Consul Metrics Integration Checks @@ -30,14 +35,20 @@ The Helm chart includes automated configuration options in order to integrate wi - | Consul Component | Description | API Endpoint(s) | - |------------------|--------------------------------------------------|----------------------------------------| - | Serf | Events and Membership Flaps | `/v1/agent/metrics` (prometheus) | - | Raft | Monitors Raft peer information | `/v1/status/leader` `/v1/status/peers` | - | Catalog Services | Service Health Status and Node Count | | - | Catalog Nodes | Node Service Count and Health Status | | - | Consul Latency | Consul LAN + WAN Coordinate Latency Calculations | | + | Consul Component | Description | API Endpoint(s) | + |------------------|--------------------------------------------------|----------------------------------------------------------------------| + | Agent | Agent Metadata (i.e., version) | `/v1/agent/self` | + | Metrics | Prometheus formatted metrics | `/v1/agent/metrics` | + | Serf | Events and Membership Flaps | `/v1/health/service/consul` `/v1/agent/self` | + | Raft | Monitors Raft peer information | `/v1/status/leader` `/v1/status/peers` | + | Catalog Services | Service Health Status and Node Count | `/v1/catalog/services` `/v1/health/state/any` | + | Catalog Nodes | Node Service Count and Health Status | `/v1/health/state/any` `/v1/health/service/` | + | Consul Latency | Consul LAN + WAN Coordinate Latency Calculations | `/v1/agent/self` `/v1/coordinate/nodes` `/v1/coordinate/datacenters` | + + +### Metrics Data Collected +Review the Datadog Documentation for the full description of Metrics data collected by the official Consul integration [here](https://docs.datadoghq.com/integrations/consul/?tab=host#data-collected). ## Consul Server Agent DogstatsD Metrics Collection From 673271ef5e62fb450251c10346b958746e6fea93 Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Wed, 14 Feb 2024 04:42:21 +0000 Subject: [PATCH 03/19] backport of commit 6435d67ea92655d48d7326cc2972bb1bc82ff65b --- .../k8s/connect/observability/datadog.mdx | 88 +++++++++++-------- 1 file changed, 53 insertions(+), 35 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 86a1a48290ef..16cb4d01329b 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -2,7 +2,7 @@ layout: docs page_title: Configure Datadog Metrics Collection for Consul on Kubernetes description: >- - Enable Consul Monitoring via Datadog using the `metrics.datadog` helm annotation. + Enable Consul Monitoring via Datadog using the `metrics.datadog` helm value override options. --- # Datadog Integration for Consul on Kubernetes @@ -12,17 +12,16 @@ The Helm chart includes automated configuration options in order to integrate wi ### Datadog Metrics Integration Methods - Choose one integration method from the three described below that best suites the intent for metrics collection. Consul Integration, DogStatsD, and Openmetrics Prometheus methods of integration are mutually exclusive. + Choose one integration method from the three described below that best suites the intent for metrics collection. DogStatsD, Consul Integration, and Openmetrics Prometheus methods of integration are mutually exclusive.

- Reasoning: The consul-k8s helm chart automated configuration implements Datadog's Consul Integration method using the use_prometheus_endpoint configuration parameter. DogstatsD, Consul Integration, and Openmetrics Prometheus Metrics by design share the same metric name syntax for collection, and would therefore cause a conflict. + Reasoning: The consul-k8s helm chart automated configuration implements Datadog's Consul Integration method using the use_prometheus_endpoint configuration parameter. DogstatsD, Consul Integration, and Openmetrics Prometheus Metrics by design share the same metric name syntax for collection, and would therefore cause a conflict. The consul.py integration source code, as well as the consul-k8s helm chart prohibit the enablement of more that one integration at a time. +## Consul Server Agent DogstatsD Metrics Collection - -## Consul Metrics Integration Checks - - + + ```yaml metrics: @@ -31,29 +30,15 @@ The Helm chart includes automated configuration options in order to integrate wi datadog: enabled: true namespace: "datadog" + dogstatsd: + enabled: true + socketTransportType: "UDS" + dogstatsdAddr: "/var/run/datadog/dsd.socket" ``` - | Consul Component | Description | API Endpoint(s) | - |------------------|--------------------------------------------------|----------------------------------------------------------------------| - | Agent | Agent Metadata (i.e., version) | `/v1/agent/self` | - | Metrics | Prometheus formatted metrics | `/v1/agent/metrics` | - | Serf | Events and Membership Flaps | `/v1/health/service/consul` `/v1/agent/self` | - | Raft | Monitors Raft peer information | `/v1/status/leader` `/v1/status/peers` | - | Catalog Services | Service Health Status and Node Count | `/v1/catalog/services` `/v1/health/state/any` | - | Catalog Nodes | Node Service Count and Health Status | `/v1/health/state/any` `/v1/health/service/` | - | Consul Latency | Consul LAN + WAN Coordinate Latency Calculations | `/v1/agent/self` `/v1/coordinate/nodes` `/v1/coordinate/datacenters` | - - -### Metrics Data Collected - -Review the Datadog Documentation for the full description of Metrics data collected by the official Consul integration [here](https://docs.datadoghq.com/integrations/consul/?tab=host#data-collected). - -## Consul Server Agent DogstatsD Metrics Collection - - - + ```yaml metrics: @@ -64,25 +49,28 @@ Review the Datadog Documentation for the full description of Metrics data collec namespace: "datadog" dogstatsd: enabled: true - socketTransportType: "UDS" - dogstatsdAddr: "/var/run/datadog/dsd.socket" + socketTransportType: "UDP" + # Set `dogstatsdPort` to `0` (default) to omit port number append to address. + dogstatsdPort: 0 + dogstatsdAddr: "datadog.datadog.svc.cluster.local" ``` - + ```yaml metrics: enabled: true enableAgentMetrics: true - datadog: + datadog: + enabled: true + namespace: "datadog" + dogstatsd: enabled: true - namespace: "datadog" - dogstatsd: - enabled: true - socketTransportType: "UDP" - dogstatsdAddr: "datadog.datadog.svc.cluster.local" + socketTransportType: "UDP" + dogstatsdPort: 8125 + dogstatsdAddr: "172.20.180.10" ``` @@ -91,6 +79,36 @@ Review the Datadog Documentation for the full description of Metrics data collec - Collects DogstatsD formatted metrics pertaining to Consul Serf Membership, Raft, DNS Performance, Agent Telemetry, and much more. - Enables configuring DogstatsD metric collection using one of either `UDP` or `Unix Domain Socket configuration` +## Datadog Official Consul Integration Checks + + + + ```yaml + metrics: + enabled: true + enableAgentMetrics: true + datadog: + enabled: true + namespace: "datadog" + ``` + + + + | Consul Component | Description | API Endpoint(s) | + |------------------|-----------------------------------------------------|----------------------------------------------------------------------| + | Agent | Agent Metadata (i.e., version) | `/v1/agent/self` | + | Metrics | Prometheus formatted metrics | `/v1/agent/metrics` | + | Serf | Events and Membership Flaps | `/v1/health/service/consul` `/v1/agent/self` | + | Raft | Monitors Raft peer information and leader elections | `/v1/status/leader` `/v1/status/peers` | + | Catalog Services | Service Health Status and Node Count | `/v1/catalog/services` `/v1/health/state/any` | + | Catalog Nodes | Node Service Count and Health Status | `/v1/health/state/any` `/v1/health/service/` | + | Consul Latency | Consul LAN + WAN Coordinate Latency Calculations | `/v1/agent/self` `/v1/coordinate/nodes` `/v1/coordinate/datacenters` | + + +### Metrics Data Collected + +Review the Datadog Documentation for the full description of Metrics data collected by the official Consul integration [here](https://docs.datadoghq.com/integrations/consul/?tab=host#data-collected). + ## Consul Server Openmetrics Prometheus Metrics Collection From 27e09daf168351da62189338da33aff879138518 Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Wed, 14 Feb 2024 23:19:31 +0000 Subject: [PATCH 04/19] backport of commit 45e5073628eefc1686972a1ce9d1cf6527b445fd --- .../k8s/connect/observability/datadog.mdx | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 16cb4d01329b..6d5dde5d2e35 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -9,7 +9,11 @@ description: >- The Helm chart includes automated configuration options in order to integrate with Datadog. -### Datadog Metrics Integration Methods +## Datadog Metrics Integration Methods + +- DogstatsD +- Datadog Checks: Official Consul Integration +- Openmetrics Prometheus Choose one integration method from the three described below that best suites the intent for metrics collection. DogStatsD, Consul Integration, and Openmetrics Prometheus methods of integration are mutually exclusive. @@ -18,7 +22,7 @@ The Helm chart includes automated configuration options in order to integrate wi The consul.py integration source code, as well as the consul-k8s helm chart prohibit the enablement of more that one integration at a time. -## Consul Server Agent DogstatsD Metrics Collection +### DogstatsD: Consul Server Agent Metrics Collection @@ -76,10 +80,12 @@ The Helm chart includes automated configuration options in order to integrate wi +#### Metrics Data Collected + - Collects DogstatsD formatted metrics pertaining to Consul Serf Membership, Raft, DNS Performance, Agent Telemetry, and much more. - Enables configuring DogstatsD metric collection using one of either `UDP` or `Unix Domain Socket configuration` -## Datadog Official Consul Integration Checks +### Datadog Checks: (Official Consul Integration) @@ -105,11 +111,13 @@ The Helm chart includes automated configuration options in order to integrate wi | Consul Latency | Consul LAN + WAN Coordinate Latency Calculations | `/v1/agent/self` `/v1/coordinate/nodes` `/v1/coordinate/datacenters` | -### Metrics Data Collected +#### Metrics Data Collected -Review the Datadog Documentation for the full description of Metrics data collected by the official Consul integration [here](https://docs.datadoghq.com/integrations/consul/?tab=host#data-collected). +The list of Consul's Prometheus metrics scraped and mapped by this method are listed in the latest [metrics.py](https://github.com/DataDog/integrations-core/blob/master/consul/datadog_checks/consul/metrics.py) of the integration source code. -## Consul Server Openmetrics Prometheus Metrics Collection +Review the [Datadog Documentation](https://docs.datadoghq.com/integrations/consul/?tab=containerized#data-collected) for the full description of Metrics data collected. + +### Openmetrics Prometheus: Consul Server Agent Metrics Collection (Advanced) @@ -126,4 +134,24 @@ Review the Datadog Documentation for the full description of Metrics data collec +This method of integration is useful for cases where Prometheus enabled scrapes are desired, and further customization of the collected data is desired. + +By default, all metrics pulled using this method will scrape Consul metrics using the `/v1/agent/metrics?format=prometheus` API query, and are considered to be **_custom metrics_**. + +Use of this method will map to Datadog as described in [Mapping Prometheus Metrics to Datadog Metrics](https://docs.datadoghq.com/integrations/guide/prometheus-metrics/?tab=latestversion). + +| OpenMetrics Metric Type | Datadog Metric Type | +|-------------------------|------------------------------------| +| `Gauge` | `gauge` | +| `Counter` | `count` | +| Histogram: `_count ` | `count.count` | +| Histogram: `_sum` | `count.sum` | +| Histogram: `_bucket` | `count.bucket` \|\| `distribution` | +| Summary: `_count` | `count.count` | +| Summary: `_sum` | `count.sum` | +| Summary: `sample` | `gauge.quantile` | + +#### Metrics Data Collected + - Collects Consul related metrics via the `/v1/agent/metrics` API endpoint leveraging the `prometheus` query parameter. + - Default configuration, uses wildcard `".*"` to collect all metrics emitted from the endpoint. From f899df49864564cdefd13448c8b62e495fcd3354 Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Thu, 15 Feb 2024 01:13:07 +0000 Subject: [PATCH 05/19] backport of commit e4a0fa97a1f46ee4381bb7956bd6084b024d83a0 --- .../k8s/connect/observability/datadog.mdx | 85 +++++++++++++++++-- 1 file changed, 77 insertions(+), 8 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 6d5dde5d2e35..597bc5a9ef0c 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -11,9 +11,9 @@ The Helm chart includes automated configuration options in order to integrate wi ## Datadog Metrics Integration Methods -- DogstatsD -- Datadog Checks: Official Consul Integration -- Openmetrics Prometheus +- [DogstatsD](#dogstatsd) +- [Datadog Checks: Official Consul Integration](#datadog-checks-official-consul-integration) +- [Openmetrics Prometheus](#openmetrics-prometheus) Choose one integration method from the three described below that best suites the intent for metrics collection. DogStatsD, Consul Integration, and Openmetrics Prometheus methods of integration are mutually exclusive. @@ -22,7 +22,7 @@ The Helm chart includes automated configuration options in order to integrate wi The consul.py integration source code, as well as the consul-k8s helm chart prohibit the enablement of more that one integration at a time. -### DogstatsD: Consul Server Agent Metrics Collection +## DogstatsD @@ -56,7 +56,7 @@ The Helm chart includes automated configuration options in order to integrate wi socketTransportType: "UDP" # Set `dogstatsdPort` to `0` (default) to omit port number append to address. dogstatsdPort: 0 - dogstatsdAddr: "datadog.datadog.svc.cluster.local" + dogstatsdAddr: "datadog-agent.datadog.svc.cluster.local" ``` @@ -80,12 +80,80 @@ The Helm chart includes automated configuration options in order to integrate wi +This integration method accomplishes metrics collection by leveraging either [Unix Domain Sockets](https://docs.datadoghq.com/developers/dogstatsd/unix_socket/?tab=kubernetes) (**UDS**) or [User Datagram Protocol](https://docs.datadoghq.com/developers/dogstatsd/?tab=kubernetes#agent) (**UDP**) transport. + + | Transport | Advantages | Disadvantages | Description | + |-----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------| + | UDS |
  • No IP or DNS resolution requirement for Datadog Agent
  • Improved network performance
  • Packet error handling
  • Automatic container ID tagging
| | Transmits DogstatsD Metrics via:
  • Unix Domain Socket
| + | UDP |
  • Does not require hostPath Volume attachment
  • Similar `IP:Port` configuration as Virtual Machine hosts
|
  • (**_Hostport_**) Requires a networking provider that adheres to the CNI specification, such as Calico, Canal, or Flannel.
  • (**_Hostport_**) Requires port to be exposed on host using `hostNetwork`
  • (**_Hostport_**) Requires firewall access controls to permit access
  • (**_Hostport_**) Network Namespace sharing is required
| Transmits DogstatsD Metrics via:
  • Container host port
    • or
  • Kubernetes Service `IP:Port`
| + #### Metrics Data Collected - Collects DogstatsD formatted metrics pertaining to Consul Serf Membership, Raft, DNS Performance, Agent Telemetry, and much more. - Enables configuring DogstatsD metric collection using one of either `UDP` or `Unix Domain Socket configuration` -### Datadog Checks: (Official Consul Integration) + +#### Verifying DogstatsD Metric Collection + + | Datadog Agent Command | Pod | Container | + |-----------------------|---------------|-----------| + | `$ agent status` | datadog-agent | agent | + + + + + ```shell + # Example: UDP Packet and Metric Packet Traffic Increase + ========= + DogStatsD + ========= + Event Packets: 0 + Event Parse Errors: 0 + Metric Packets: 5,908 + Metric Parse Errors: 0 + Service Check Packets: 0 + Service Check Parse Errors: 0 + Udp Bytes: 636,872 + Udp Packet Reading Errors: 0 + Udp Packets: 3,300 + Uds Bytes: 0 + Uds Origin Detection Errors: 0 + Uds Packet Reading Errors: 0 + Uds Packets: 0 + Unterminated Metric Errors: 0 + ``` + + + + ```shell + # Example: UDS Packet and Metric Packet Traffic Increase + ========= + DogStatsD + ========= + Event Packets: 0 + Event Parse Errors: 0 + Metric Packets: 30,523 + Metric Parse Errors: 0 + Service Check Packets: 0 + Service Check Parse Errors: 0 + Udp Bytes: 124,635 + Udp Packet Reading Errors: 0 + Udp Packets: 731 + Uds Bytes: 2,957,433 + Uds Origin Detection Errors: 0 + Uds Packet Reading Errors: 0 + Uds Packets: 11,563 + Unterminated Metric Errors: 0 + ``` + + + + + + + + +## Datadog Checks: Official Consul Integration @@ -117,7 +185,7 @@ The list of Consul's Prometheus metrics scraped and mapped by this method are li Review the [Datadog Documentation](https://docs.datadoghq.com/integrations/consul/?tab=containerized#data-collected) for the full description of Metrics data collected. -### Openmetrics Prometheus: Consul Server Agent Metrics Collection (Advanced) +## Openmetrics Prometheus @@ -138,7 +206,7 @@ This method of integration is useful for cases where Prometheus enabled scrapes By default, all metrics pulled using this method will scrape Consul metrics using the `/v1/agent/metrics?format=prometheus` API query, and are considered to be **_custom metrics_**. -Use of this method will map to Datadog as described in [Mapping Prometheus Metrics to Datadog Metrics](https://docs.datadoghq.com/integrations/guide/prometheus-metrics/?tab=latestversion). +Use of this method will map to Datadog as described in [Mapping Prometheus Metrics to Datadog Metrics](https://docs.datadoghq.com/integrations/guide/prometheus-metrics/?tab=latestversion) and summarized below. | OpenMetrics Metric Type | Datadog Metric Type | |-------------------------|------------------------------------| @@ -151,6 +219,7 @@ Use of this method will map to Datadog as described in [Mapping Prometheus Metri | Summary: `_sum` | `count.sum` | | Summary: `sample` | `gauge.quantile` | + #### Metrics Data Collected - Collects Consul related metrics via the `/v1/agent/metrics` API endpoint leveraging the `prometheus` query parameter. From 395bdce40fe88bfeadb6192feed6f9d2d6268b59 Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Thu, 15 Feb 2024 01:15:10 +0000 Subject: [PATCH 06/19] backport of commit 8ac1b1bd26fbccaa43867d74a07880e970a3285e --- website/content/docs/k8s/connect/observability/datadog.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 597bc5a9ef0c..2329ed515d78 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -42,7 +42,7 @@ The Helm chart includes automated configuration options in order to integrate wi - + ```yaml metrics: @@ -61,7 +61,7 @@ The Helm chart includes automated configuration options in order to integrate wi - + ```yaml metrics: From 10bb49d4ff8b956910661a11db2789a0dd481444 Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Thu, 15 Feb 2024 18:22:13 +0000 Subject: [PATCH 07/19] backport of commit c8084f42054bd635f0262042b4ad4c2e8a186aed --- .../k8s/connect/observability/datadog.mdx | 54 +++++++++++++++---- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 2329ed515d78..dccbc580c091 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -16,7 +16,7 @@ The Helm chart includes automated configuration options in order to integrate wi - [Openmetrics Prometheus](#openmetrics-prometheus) - Choose one integration method from the three described below that best suites the intent for metrics collection. DogStatsD, Consul Integration, and Openmetrics Prometheus methods of integration are mutually exclusive. + Users should choose one integration method from the three described below that best suites the intent for metrics collection. DogStatsD, Consul Integration, and Openmetrics Prometheus methods of integration are mutually exclusive.

Reasoning: The consul-k8s helm chart automated configuration implements Datadog's Consul Integration method using the use_prometheus_endpoint configuration parameter. DogstatsD, Consul Integration, and Openmetrics Prometheus Metrics by design share the same metric name syntax for collection, and would therefore cause a conflict. The consul.py integration source code, as well as the consul-k8s helm chart prohibit the enablement of more that one integration at a time. @@ -24,6 +24,8 @@ The Helm chart includes automated configuration options in order to integrate wi ## DogstatsD + + @@ -82,16 +84,44 @@ The Helm chart includes automated configuration options in order to integrate wi This integration method accomplishes metrics collection by leveraging either [Unix Domain Sockets](https://docs.datadoghq.com/developers/dogstatsd/unix_socket/?tab=kubernetes) (**UDS**) or [User Datagram Protocol](https://docs.datadoghq.com/developers/dogstatsd/?tab=kubernetes#agent) (**UDP**) transport. - | Transport | Advantages | Disadvantages | Description | - |-----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------| - | UDS |
  • No IP or DNS resolution requirement for Datadog Agent
  • Improved network performance
  • Packet error handling
  • Automatic container ID tagging
| | Transmits DogstatsD Metrics via:
  • Unix Domain Socket
| - | UDP |
  • Does not require hostPath Volume attachment
  • Similar `IP:Port` configuration as Virtual Machine hosts
|
  • (**_Hostport_**) Requires a networking provider that adheres to the CNI specification, such as Calico, Canal, or Flannel.
  • (**_Hostport_**) Requires port to be exposed on host using `hostNetwork`
  • (**_Hostport_**) Requires firewall access controls to permit access
  • (**_Hostport_**) Network Namespace sharing is required
| Transmits DogstatsD Metrics via:
  • Container host port
    • or
  • Kubernetes Service `IP:Port`
| - -#### Metrics Data Collected - - - Collects DogstatsD formatted metrics pertaining to Consul Serf Membership, Raft, DNS Performance, Agent Telemetry, and much more. - - Enables configuring DogstatsD metric collection using one of either `UDP` or `Unix Domain Socket configuration` - + + + + + + + + + + + + + + + + + +
AdvantagesDisadvantagesPacket Delivery
No IP or DNS resolution requirement for Datadog Agent

Improved network performance

Packet error handling

Automatic container ID tagging		Requires hostPath Volume attachmentUnix Domain Socket File
+ + + + + + + + + + + + + + + + + +
AdvantagesDisadvantagesPacket Delivery
Does not require hostPath Volume attachment

Similar IP:Port configuration as Virtual Machine hosts		(Hostport) Requires a networking provider that adheres to the CNI specification, such as Calico, Canal, or Flannel.

(Hostport) Requires port to be exposed on host using hostNetwork

(Hostport) Requires firewall access controls to permit access

(Hostport) Network Namespace sharing is required		Kubernetes Service IP:Port

or

Container host port
+ + #### Verifying DogstatsD Metric Collection @@ -150,7 +180,9 @@ This integration method accomplishes metrics collection by leveraging either [Un +#### Metrics Data Collected + - Full list of metrics sent via DogstatsD consists of those listed in the [Agent Telemetry](https://developer.hashicorp.com/consul/docs/agent/telemetry) documentation. ## Datadog Checks: Official Consul Integration From 873af4611f375cd70969c847e33ab52d9f10f078 Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Thu, 15 Feb 2024 20:01:13 +0000 Subject: [PATCH 08/19] backport of commit 41577dab220ef6a61e869328810c865affa416f2 --- .../k8s/connect/observability/datadog.mdx | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index dccbc580c091..30d507055aae 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -24,11 +24,19 @@ The Helm chart includes automated configuration options in order to integrate wi ## DogstatsD +This method of implementation leverages the [hashicorp/go-metrics DogstatsD client library](https://github.com/hashicorp/go-metrics/tree/master/datadog) to manage metrics collection. +Metrics are aggregated and sent via UDP or UDS transports to a Datadog Agent that runs on the same Kube Node as the Consul servers. +Enabling this method of metrics collection allows Consul to control the delivery of metrics traffic directly to a Datadog agent rather +than a Datadog agent attempting to reach Consul and scrape the `/v1/agent/metrics` API endpoint. + +This is accomplished by updating each server agent's configuration telemetry stanza. + Consul Helm Chart Overrides + ```yaml metrics: enabled: true @@ -42,10 +50,22 @@ The Helm chart includes automated configuration options in order to integrate wi dogstatsdAddr: "/var/run/datadog/dsd.socket" ``` + Resulting server agent telemetry configuration + + ```json + { + "telemetry": { + "dogstatsd_addr": "unix:///var/run/datadog/dsd.socket" + } + } + ``` + + Consul Helm Chart Overrides + ```yaml metrics: enabled: true @@ -56,15 +76,26 @@ The Helm chart includes automated configuration options in order to integrate wi dogstatsd: enabled: true socketTransportType: "UDP" - # Set `dogstatsdPort` to `0` (default) to omit port number append to address. - dogstatsdPort: 0 + dogstatsdPort: 0 # Set `dogstatsdPort` to `0` (default) to omit port number append to address. dogstatsdAddr: "datadog-agent.datadog.svc.cluster.local" ``` + Resulting server agent telemetry configuration + + ```json + { + "telemetry": { + "dogstatsd_addr": "datadog-agent.datadog.svc.cluster.local" + } + } + ``` + + Consul Helm Chart Overrides + ```yaml metrics: enabled: true @@ -79,6 +110,16 @@ The Helm chart includes automated configuration options in order to integrate wi dogstatsdAddr: "172.20.180.10" ``` + Resulting server agent telemetry configuration + + ```json + { + "telemetry": { + "dogstatsd_addr": "172.20.180.10:8125", + } + } + ``` + @@ -187,6 +228,7 @@ This integration method accomplishes metrics collection by leveraging either [Un ## Datadog Checks: Official Consul Integration + ```yaml @@ -215,6 +257,8 @@ This integration method accomplishes metrics collection by leveraging either [Un The list of Consul's Prometheus metrics scraped and mapped by this method are listed in the latest [metrics.py](https://github.com/DataDog/integrations-core/blob/master/consul/datadog_checks/consul/metrics.py) of the integration source code. +To understand how Consul Latency metrics are calculated, review the [Consul Network Coordinates](https://developer.hashicorp.com/consul/docs/architecture/coordinates) documentation. + Review the [Datadog Documentation](https://docs.datadoghq.com/integrations/consul/?tab=containerized#data-collected) for the full description of Metrics data collected. ## Openmetrics Prometheus @@ -255,4 +299,4 @@ Use of this method will map to Datadog as described in [Mapping Prometheus Metri #### Metrics Data Collected - Collects Consul related metrics via the `/v1/agent/metrics` API endpoint leveraging the `prometheus` query parameter. - - Default configuration, uses wildcard `".*"` to collect all metrics emitted from the endpoint. + - Default configuration, uses a wildcard `".*"` to collect all metrics emitted from the endpoint, therefore the list of metrics scraped consists of those listed in the [Agent Telemetry](https://developer.hashicorp.com/consul/docs/agent/telemetry) documentation. From 564449ee8f5e72c2d76354eed4cae38de584bca2 Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Thu, 15 Feb 2024 21:19:06 +0000 Subject: [PATCH 09/19] backport of commit c5b41671d3bd2ead9076ecf0646c1efe1be477bc --- .../k8s/connect/observability/datadog.mdx | 59 ++++++++++++++----- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 30d507055aae..f17d083ffa82 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -32,6 +32,8 @@ than a Datadog agent attempting to reach Consul and scrape the `/v1/agent/metric This is accomplished by updating each server agent's configuration telemetry stanza. +### Helm Chart Configuration + @@ -124,6 +126,9 @@ This is accomplished by updating each server agent's configuration telemetry sta This integration method accomplishes metrics collection by leveraging either [Unix Domain Sockets](https://docs.datadoghq.com/developers/dogstatsd/unix_socket/?tab=kubernetes) (**UDS**) or [User Datagram Protocol](https://docs.datadoghq.com/developers/dogstatsd/?tab=kubernetes#agent) (**UDP**) transport. +Practitioners who manage their Kubernetes infrastructure and/or service-mesh should take into account the implications outlined in the table below. + +### UDS/UDP Advantages and Disadvantages @@ -137,8 +142,8 @@ This integration method accomplishes metrics collection by leveraging either [Un - No IP or DNS resolution requirement for Datadog Agent

Improved network performance

Packet error handling

Automatic container ID tagging - Requires hostPath Volume attachment + No IP or DNS resolution requirement for Datadog Agent

Improved network performance

Higher throughput capacity

Packet error handling

Automatic container ID tagging + Requires hostPath Volume attachment

Datadog Agent must run on every host you send metrics from Unix Domain Socket File @@ -155,8 +160,8 @@ This integration method accomplishes metrics collection by leveraging either [Un - Does not require hostPath Volume attachment

Similar IP:Port configuration as Virtual Machine hosts - (Hostport) Requires a networking provider that adheres to the CNI specification, such as Calico, Canal, or Flannel.

(Hostport) Requires port to be exposed on host using hostNetwork

(Hostport) Requires firewall access controls to permit access

(Hostport) Network Namespace sharing is required + Does not require hostPath Volume attachment

(KubeDNS) Does not require Hostport exposure if accessible from cluster

Similar IP:Port configuration as Virtual Machine hosts + No packet error handling

(Hostport) Requires a networking provider that adheres to the CNI specification, such as Calico, Canal, or Flannel.

(Hostport) Requires port to be exposed on host using hostNetwork

(Hostport) Requires firewall access controls to permit access

(Hostport) Network Namespace sharing is required Kubernetes Service IP:Port

or

Container host port @@ -166,9 +171,16 @@ This integration method accomplishes metrics collection by leveraging either [Un #### Verifying DogstatsD Metric Collection - | Datadog Agent Command | Pod | Container | - |-----------------------|---------------|-----------| - | `$ agent status` | datadog-agent | agent | +To confirm you're Datadog agent is receiving traffic, the `status` subcommand can be ran from the Datadog Agent expecting to receive DogstatsD traffic from Consul. + +There should be an increase in either UDP or UDS traffic packet counts from the resultant output after the configuration has been properly established. + + + | Transport | Command | Pod | Container | Example Output | + |:------------|----------------------------------------------------------------------------|---------------|-----------|------------------------------------------------------------------------------------------------| + | `UDP`/`UDS` | `agent status` | datadog-agent | agent | See below. | + | `UDP` | `netstat -nup \| grep ":8125.*ESTABLISHED"` | consul-server | consul | `udp 0 0 127.0.0.1:53874 127.0.0.1:8125 ESTABLISHED 23176/consul` | + | `UDS` | `nc -U -u -w1 /var/run/datadog/dsd.socket` _(update socket path as req'd)_ | consul-server | consul | `Bound on /tmp/nc-IjJkoG/recv.sock` | @@ -228,6 +240,27 @@ This integration method accomplishes metrics collection by leveraging either [Un ## Datadog Checks: Official Consul Integration +The Datadog Agent package includes official third-party integrations for built-in availability upon agent deployment. + +The Consul Integration Datadog checks provided some additional metric verification checks that leverage Consul's built-in feature-set, and help monitor Consul +during normal operation beyond that of Consul's available metrics. + +See the below table for an outline of the features added by the official integration. + +### Additional Integration Checks Performed + +| Consul Component | Description | API Endpoint(s) | +|------------------|-----------------------------------------------------|----------------------------------------------------------------------| +| Agent | Agent Metadata (i.e., version) | `/v1/agent/self` | +| Metrics | Prometheus formatted metrics | `/v1/agent/metrics` | +| Serf | Events and Membership Flaps | `/v1/health/service/consul` `/v1/agent/self` | +| Raft | Monitors Raft peer information and leader elections | `/v1/status/leader` `/v1/status/peers` | +| Catalog Services | Service Health Status and Node Count | `/v1/catalog/services` `/v1/health/state/any` | +| Catalog Nodes | Node Service Count and Health Status | `/v1/health/state/any` `/v1/health/service/` | +| Consul Latency | Consul LAN + WAN Coordinate Latency Calculations | `/v1/agent/self` `/v1/coordinate/nodes` `/v1/coordinate/datacenters` | + + +### Helm Chart Configuration @@ -242,15 +275,7 @@ This integration method accomplishes metrics collection by leveraging either [Un - | Consul Component | Description | API Endpoint(s) | - |------------------|-----------------------------------------------------|----------------------------------------------------------------------| - | Agent | Agent Metadata (i.e., version) | `/v1/agent/self` | - | Metrics | Prometheus formatted metrics | `/v1/agent/metrics` | - | Serf | Events and Membership Flaps | `/v1/health/service/consul` `/v1/agent/self` | - | Raft | Monitors Raft peer information and leader elections | `/v1/status/leader` `/v1/status/peers` | - | Catalog Services | Service Health Status and Node Count | `/v1/catalog/services` `/v1/health/state/any` | - | Catalog Nodes | Node Service Count and Health Status | `/v1/health/state/any` `/v1/health/service/` | - | Consul Latency | Consul LAN + WAN Coordinate Latency Calculations | `/v1/agent/self` `/v1/coordinate/nodes` `/v1/coordinate/datacenters` | + #### Metrics Data Collected @@ -263,6 +288,8 @@ Review the [Datadog Documentation](https://docs.datadoghq.com/integrations/consu ## Openmetrics Prometheus +### Helm Chart Configuration + ```yaml From 322119f837dd8ee580e4a5fa5140f4a0dcc30b52 Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Fri, 16 Feb 2024 20:02:40 +0000 Subject: [PATCH 10/19] backport of commit bddbd0d9229eb90f3dd921bb89fa558d0f605aee --- .../k8s/connect/observability/datadog.mdx | 64 ++++++++----------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index f17d083ffa82..6cbd1956cc10 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -130,44 +130,32 @@ Practitioners who manage their Kubernetes infrastructure and/or service-mesh sho ### UDS/UDP Advantages and Disadvantages - - - - - - - - - - - - - - - - - -
AdvantagesDisadvantagesPacket Delivery
No IP or DNS resolution requirement for Datadog Agent

Improved network performance

Higher throughput capacity

Packet error handling

Automatic container ID tagging		Requires hostPath Volume attachment

Datadog Agent must run on every host you send metrics from		Unix Domain Socket File
- - - - - - - - - - - - - - - - - -
AdvantagesDisadvantagesPacket Delivery
Does not require hostPath Volume attachment

(KubeDNS) Does not require Hostport exposure if accessible from cluster

Similar IP:Port configuration as Virtual Machine hosts		No packet error handling

(Hostport) Requires a networking provider that adheres to the CNI specification, such as Calico, Canal, or Flannel.

(Hostport) Requires port to be exposed on host using hostNetwork

(Hostport) Requires firewall access controls to permit access

(Hostport) Network Namespace sharing is required		Kubernetes Service IP:Port

or

Container host port
- - +## UDS + +**Packet Transport**: Unix Domain Socket File + +| Advantages | Disadvantages | +|-------------------------------------------------------|------------------------------------------------------------------------------------------------------| +| No IP or DNS resolution requirement for Datadog Agent | Requires [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) Volume attachment | +| Improved network performance | Datadog Agent must run on every host you send metrics from | +| Higher throughput capacity | | +| Packet error handling | | +| Automatic container ID tagging | | + +## UDP + +**Packet Transport**: + * Kubernetes Service `IP:Port` + * Container Host Port + +| Advantages | Disadvantages | +|------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------| +| Does **not** require [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) Volume attachment | **No** packet error handling | +| (**_KubeDNS_**) Does **not** require Hostport exposure if accessible from cluster | (**_Hostport_**) Requires a networking provider that adheres to the CNI specification, such as Calico, Canal, or Flannel. | +| Similar `IP:Port` configuration as Virtual Machine hosts | (**_Hostport_**) Requires port to be exposed on host using `hostNetwork` | +| | (**_Hostport_**) Requires firewall access controls to permit access | +| | (**_Hostport_**) Network Namespace sharing is required | + #### Verifying DogstatsD Metric Collection From 87a7dd97831e42bc9ab0cada8b81a794821c8e3b Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Fri, 16 Feb 2024 23:32:58 +0000 Subject: [PATCH 11/19] backport of commit 57993715ae069dd314ee2c182e569940166eec38 --- .../k8s/connect/observability/datadog.mdx | 195 +++++++++++++----- 1 file changed, 149 insertions(+), 46 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 6cbd1956cc10..d2ff21d2fb77 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -15,12 +15,10 @@ The Helm chart includes automated configuration options in order to integrate wi - [Datadog Checks: Official Consul Integration](#datadog-checks-official-consul-integration) - [Openmetrics Prometheus](#openmetrics-prometheus) - - Users should choose one integration method from the three described below that best suites the intent for metrics collection. DogStatsD, Consul Integration, and Openmetrics Prometheus methods of integration are mutually exclusive. -

- Reasoning: The consul-k8s helm chart automated configuration implements Datadog's Consul Integration method using the use_prometheus_endpoint configuration parameter. DogstatsD, Consul Integration, and Openmetrics Prometheus Metrics by design share the same metric name syntax for collection, and would therefore cause a conflict. - The consul.py integration source code, as well as the consul-k8s helm chart prohibit the enablement of more that one integration at a time. - +Users should choose **_one_** integration method from the three described below that best suites the intent for metrics collection. **[DogStatsD](https://docs.datadoghq.com/developers/dogstatsd/?tab=kubernetes)**, **[Consul Integration](https://docs.datadoghq.com/integrations/consul/?tab=containerized)**, and **[Openmetrics Prometheus](https://docs.datadoghq.com/containers/kubernetes/prometheus/?tab=kubernetesadv2)** methods of integration are **_mutually exclusive_**. + +**Reasoning:** _The consul-k8s helm chart automated configuration implements Datadog's [Consul Integration](https://docs.datadoghq.com/integrations/consul/?tab=containerized) method using the [`use_prometheus_endpoint`](https://github.com/DataDog/integrations-core/blob/07c04c5e9465ba1f3e0198830896d05923e81283/consul/datadog_checks/consul/data/conf.yaml.example#L59) configuration parameter. **DogstatsD**, **Consul Integration**, and **Openmetrics Prometheus** Metrics **by design** share the same [metric name](https://docs.datadoghq.com/integrations/consul/?tab=host#data-collected) syntax for collection, and would therefore cause a conflict. The [consul.py](https://github.com/DataDog/integrations-core/blob/07c04c5e9465ba1f3e0198830896d05923e81283/consul/datadog_checks/consul/consul.py#L55-L61) integration source code, as well as the [consul-k8s helm chart](https://github.com/hashicorp/consul-k8s/blob/4cac70496788f50354f96e9331003fcf338f419c/charts/consul/templates/_helpers.tpl#L595-L598) prohibit the enablement of more that one integration at a time._ + ## DogstatsD @@ -78,7 +76,8 @@ This is accomplished by updating each server agent's configuration telemetry sta dogstatsd: enabled: true socketTransportType: "UDP" - dogstatsdPort: 0 # Set `dogstatsdPort` to `0` (default) to omit port number append to address. + # Set `dogstatsdPort` to `0` (default) to omit port number append to address. + dogstatsdPort: 0 dogstatsdAddr: "datadog-agent.datadog.svc.cluster.local" ``` @@ -125,12 +124,12 @@ This is accomplished by updating each server agent's configuration telemetry sta -This integration method accomplishes metrics collection by leveraging either [Unix Domain Sockets](https://docs.datadoghq.com/developers/dogstatsd/unix_socket/?tab=kubernetes) (**UDS**) or [User Datagram Protocol](https://docs.datadoghq.com/developers/dogstatsd/?tab=kubernetes#agent) (**UDP**) transport. -Practitioners who manage their Kubernetes infrastructure and/or service-mesh should take into account the implications outlined in the table below. - ### UDS/UDP Advantages and Disadvantages -## UDS +This integration method accomplishes metrics collection by leveraging either [Unix Domain Sockets](https://docs.datadoghq.com/developers/dogstatsd/unix_socket/?tab=kubernetes) (**UDS**) or [User Datagram Protocol](https://docs.datadoghq.com/developers/dogstatsd/?tab=kubernetes#agent) (**UDP**) transport. +Practitioners who manage their Kubernetes infrastructure and/or service-mesh should take into account the implications outlined in the tables below. + +#### UDS **Packet Transport**: Unix Domain Socket File @@ -142,11 +141,11 @@ Practitioners who manage their Kubernetes infrastructure and/or service-mesh sho | Packet error handling | | | Automatic container ID tagging | | -## UDP +#### UDP **Packet Transport**: - * Kubernetes Service `IP:Port` - * Container Host Port + * Kubernetes Service `IP:Port` + * Container Host Port | Advantages | Disadvantages | |------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------| @@ -164,11 +163,10 @@ To confirm you're Datadog agent is receiving traffic, the `status` subcommand ca There should be an increase in either UDP or UDS traffic packet counts from the resultant output after the configuration has been properly established. - | Transport | Command | Pod | Container | Example Output | - |:------------|----------------------------------------------------------------------------|---------------|-----------|------------------------------------------------------------------------------------------------| - | `UDP`/`UDS` | `agent status` | datadog-agent | agent | See below. | - | `UDP` | `netstat -nup \| grep ":8125.*ESTABLISHED"` | consul-server | consul | `udp 0 0 127.0.0.1:53874 127.0.0.1:8125 ESTABLISHED 23176/consul` | - | `UDS` | `nc -U -u -w1 /var/run/datadog/dsd.socket` _(update socket path as req'd)_ | consul-server | consul | `Bound on /tmp/nc-IjJkoG/recv.sock` | + | Transport | Command | Pod | Container | + |:---------------|----------------------------------------------------------------------------|---------------|-----------| + | `UDP`\|\|`UDS` | `agent status` | datadog-agent | agent | + @@ -220,6 +218,65 @@ There should be an increase in either UDP or UDS traffic packet counts from the +Traffic verification can also be accomplished using the `netstat` command line utility from a consul-server expected to be submitting +metrics data to Datadog. + + + Using netstat requires privileged container permissions to install open-bsd networking tools on the consul-server for testing. + + +| Transport | Command | Pod | Container | +|:-----------------|-----------|---------------|-----------| +| `UDP` \|\| `UDS` | `netstat` | consul-server | consul | + + + + + ```shell + $ netstat -nup | grep "172.28.13.12:8125.*ESTABLISHED + udp 0 0 127.0.0.1:53874 127.0.0.1:8125 ESTABLISHED 23176/consul + ``` + + + + + + ```shell + $ netstat -x + Active UNIX domain sockets (w/o servers) + Proto RefCnt Flags Type State I-Node Path + unix 2 [ ] DGRAM CONNECTED 15952473 + unix 2 [ ] DGRAM 15652537 @9d10c + ``` + + + + + +UDS provides the additional capability for verification by sending a test metrics packet to the Unix Socket configured. + + + Using netcat (nc) requires privileged container permissions to install open-bsd networking tools on the consul-server for testing. + + +| Transport | Command | Pod | Container | +|:----------|---------|---------------|-----------| +| `UDS` | `nc` | consul-server | consul | + + + + ```shell + $ echo -n "custom.metric.name:1|c" | nc -U -u -w1 /var/run/datadog/dsd.socket + Bound on /tmp/nc-IjJkoG/recv.sock + ``` + + + +#### Use Case + +DogstatsD integration provides full-scope metrics collection from Consul, and minimizes access control configuration requirements as traffic +flow is outbound (toward the Datadog Agent) as opposed to inbound (toward the `/v1/agent/metrics/` API endpoint). + #### Metrics Data Collected @@ -233,24 +290,11 @@ The Datadog Agent package includes official third-party integrations for built-i The Consul Integration Datadog checks provided some additional metric verification checks that leverage Consul's built-in feature-set, and help monitor Consul during normal operation beyond that of Consul's available metrics. -See the below table for an outline of the features added by the official integration. - -### Additional Integration Checks Performed - -| Consul Component | Description | API Endpoint(s) | -|------------------|-----------------------------------------------------|----------------------------------------------------------------------| -| Agent | Agent Metadata (i.e., version) | `/v1/agent/self` | -| Metrics | Prometheus formatted metrics | `/v1/agent/metrics` | -| Serf | Events and Membership Flaps | `/v1/health/service/consul` `/v1/agent/self` | -| Raft | Monitors Raft peer information and leader elections | `/v1/status/leader` `/v1/status/peers` | -| Catalog Services | Service Health Status and Node Count | `/v1/catalog/services` `/v1/health/state/any` | -| Catalog Nodes | Node Service Count and Health Status | `/v1/health/state/any` `/v1/health/service/` | -| Consul Latency | Consul LAN + WAN Coordinate Latency Calculations | `/v1/agent/self` `/v1/coordinate/nodes` `/v1/coordinate/datacenters` | - +See the below [table](#additional-integration-checks-performed) for an outline of the features added by the official integration. ### Helm Chart Configuration - + ```yaml metrics: @@ -261,10 +305,23 @@ See the below table for an outline of the features added by the official integra namespace: "datadog" ``` - + +### Additional Integration Checks Performed + +| Consul Component | Description | API Endpoint(s) | +|------------------|-----------------------------------------------------|----------------------------------------------------------------------| +| Agent | Agent Metadata (i.e., version) | `/v1/agent/self` | +| Metrics | Prometheus formatted metrics | `/v1/agent/metrics` | +| Serf | Events and Membership Flaps | `/v1/health/service/consul` `/v1/agent/self` | +| Raft | Monitors Raft peer information and leader elections | `/v1/status/leader` `/v1/status/peers` | +| Catalog Services | Service Health Status and Node Count | `/v1/catalog/services` `/v1/health/state/any` | +| Catalog Nodes | Node Service Count and Health Status | `/v1/health/state/any` `/v1/health/service/` | +| Consul Latency | Consul LAN + WAN Coordinate Latency Calculations | `/v1/agent/self` `/v1/coordinate/nodes` `/v1/coordinate/datacenters` | +#### Use Case +This integration is primarily for basic Consul monitoring with focus on the service discovery. #### Metrics Data Collected @@ -276,31 +333,76 @@ Review the [Datadog Documentation](https://docs.datadoghq.com/integrations/consu ## Openmetrics Prometheus +For Datadog agents at or above v6.5.0, OpenMetrics and Prometheus checks are available to scrape Kubernetes application Prometheus endpoints. + +This method implements the collection via Openmetrics as that is fully supported for Prometheus text format and is accomplished using pod annotations as demonstrated below. + + + Enabling OpenMetrics collection via Datadog by design removes the prometheus.io/path and prometheus.io/port annotations from the consul-server statefulset deployment to allow Datadog + to scrape the agent's metrics API endpoint using either RPC TLS and Consul ACLs as necessary. + + ### Helm Chart Configuration + + Consul Helm Chart Overrides + ```yaml - metrics: - enabled: true - enableAgentMetrics: true - datadog: + global: + tls: enabled: true - namespace: "datadog" - openMetricsPrometheus: + enableAutoEncrypt: true + acls: + manageSystemACLs: true + metrics: + enabled: true + enableAgentMetrics: true + datadog: enabled: true + namespace: "datadog" + openMetricsPrometheus: + enabled: true + ``` + + Consul `server-statefulset.yaml` annotations + + ```yaml + ad.datadoghq.com/consul.checks: | + { + "openmetrics": { + "init_config": {}, + "instances": [ + { + "openmetrics_endpoint": "https://consul-server.consul.svc:8501/v1/agent/metrics?format=prometheus", + "tls_cert": "/etc/datadog-agent/conf.d/consul.d/certs/tls.crt", + "tls_private_key": "/etc/datadog-agent/conf.d/consul.d/certs/tls.key", + "tls_ca_cert": "/etc/datadog-agent/conf.d/consul.d/ca/tls.crt", + "headers": { + "X-Consul-Token": "ENC[k8s_secret@consul/consul-datadog-agent-metrics-acl-token/token]" + }, + "namespace": "consul", + "metrics": [ ".*" ] + } + ] + } + } ``` + + +#### Use Case This method of integration is useful for cases where Prometheus enabled scrapes are desired, and further customization of the collected data is desired. By default, all metrics pulled using this method will scrape Consul metrics using the `/v1/agent/metrics?format=prometheus` API query, and are considered to be **_custom metrics_**. -Use of this method will map to Datadog as described in [Mapping Prometheus Metrics to Datadog Metrics](https://docs.datadoghq.com/integrations/guide/prometheus-metrics/?tab=latestversion) and summarized below. +Use of this method will map to Datadog as described in [Mapping Prometheus Metrics to Datadog Metrics](https://docs.datadoghq.com/integrations/guide/prometheus-metrics/?tab=latestversion) and is summarized in the table below. -| OpenMetrics Metric Type | Datadog Metric Type | -|-------------------------|------------------------------------| +| OpenMetrics metric type | Datadog metric type | +|:------------------------|:-----------------------------------| | `Gauge` | `gauge` | | `Counter` | `count` | | Histogram: `_count ` | `count.count` | @@ -313,5 +415,6 @@ Use of this method will map to Datadog as described in [Mapping Prometheus Metri #### Metrics Data Collected - - Collects Consul related metrics via the `/v1/agent/metrics` API endpoint leveraging the `prometheus` query parameter. - - Default configuration, uses a wildcard `".*"` to collect all metrics emitted from the endpoint, therefore the list of metrics scraped consists of those listed in the [Agent Telemetry](https://developer.hashicorp.com/consul/docs/agent/telemetry) documentation. +The integration, by default, uses a wildcard (`".*"`) to collect **_all_** metrics emitted from the `/v1/agent/metrics` endpoint. + +Please refer to the [Agent Telemetry](https://developer.hashicorp.com/consul/docs/agent/telemetry) documentation for a full list and desription of the metrics data collected. From f98dac003017ffe964df4e9cc45b9dd1985d948a Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Fri, 16 Feb 2024 23:38:10 +0000 Subject: [PATCH 12/19] backport of commit a926b12686a9cd7740b6047bf9064fb974d8a46b --- website/content/docs/k8s/connect/observability/datadog.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index d2ff21d2fb77..5f11bf387f4b 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -399,7 +399,7 @@ This method of integration is useful for cases where Prometheus enabled scrapes By default, all metrics pulled using this method will scrape Consul metrics using the `/v1/agent/metrics?format=prometheus` API query, and are considered to be **_custom metrics_**. -Use of this method will map to Datadog as described in [Mapping Prometheus Metrics to Datadog Metrics](https://docs.datadoghq.com/integrations/guide/prometheus-metrics/?tab=latestversion) and is summarized in the table below. +Use of this method maps to Datadog as described in [Mapping Prometheus Metrics to Datadog Metrics](https://docs.datadoghq.com/integrations/guide/prometheus-metrics/?tab=latestversion). The following table summarizing how these metrics map to each other. | OpenMetrics metric type | Datadog metric type | |:------------------------|:-----------------------------------| From de8ccae016a5a4b8efb5f509e767de517a7e132d Mon Sep 17 00:00:00 2001 From: natemollica-dev <57850649+natemollica-nm@users.noreply.github.com> Date: Fri, 16 Feb 2024 23:34:36 +0000 Subject: [PATCH 13/19] backport of commit b1d3c3e6a9aecb4b2977e322d4177d98488dc61f --- website/content/docs/k8s/connect/observability/datadog.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 5f11bf387f4b..2be5c6114558 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -1,8 +1,8 @@ --- layout: docs -page_title: Configure Datadog Metrics Collection for Consul on Kubernetes +page_title: Configure Datadog metrics collection for Consul on Kubernetes description: >- - Enable Consul Monitoring via Datadog using the `metrics.datadog` helm value override options. + Consul can integrate with external platforms such as Datadog to stream metrics about its operations. Learn how to enable Consul monitoring with Datadog by configuring the `metrics.datadog` Helm value override options. --- # Datadog Integration for Consul on Kubernetes From 43417c9072cfc622aea33eecbded45f5e0da2e27 Mon Sep 17 00:00:00 2001 From: natemollica-dev <57850649+natemollica-nm@users.noreply.github.com> Date: Fri, 16 Feb 2024 23:34:57 +0000 Subject: [PATCH 14/19] backport of commit 83ca3690ad077d846c69d09ff032e187c98dbe41 --- website/content/docs/k8s/connect/observability/datadog.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 2be5c6114558..706d96072efd 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -5,7 +5,7 @@ description: >- Consul can integrate with external platforms such as Datadog to stream metrics about its operations. Learn how to enable Consul monitoring with Datadog by configuring the `metrics.datadog` Helm value override options. --- -# Datadog Integration for Consul on Kubernetes +# Configure Datadog metrics collection for Consul on Kubernetes The Helm chart includes automated configuration options in order to integrate with Datadog. From d62841a8d5309059be07a7fc77c4bf13cb8ea0cc Mon Sep 17 00:00:00 2001 From: natemollica-dev <57850649+natemollica-nm@users.noreply.github.com> Date: Fri, 16 Feb 2024 23:35:28 +0000 Subject: [PATCH 15/19] backport of commit c821820b34791af61023fb0f569419db9153b133 --- website/content/docs/k8s/connect/observability/datadog.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index 706d96072efd..fcbdff7ffd19 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -7,7 +7,7 @@ description: >- # Configure Datadog metrics collection for Consul on Kubernetes -The Helm chart includes automated configuration options in order to integrate with Datadog. +This page describes the processes for integrating Datadog metrics collection in your Consul on Kubernetes deployment. The Helm chart includes automated configuration options to simplify the integration process. ## Datadog Metrics Integration Methods From 141cadeeb85e3f78cacaa338276db913020f8f51 Mon Sep 17 00:00:00 2001 From: natemollica-dev <57850649+natemollica-nm@users.noreply.github.com> Date: Fri, 16 Feb 2024 23:35:57 +0000 Subject: [PATCH 16/19] backport of commit 54ecce7ffc0f0db8ceb8aac574c9956874f1b2c9 --- website/content/docs/k8s/connect/observability/datadog.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/connect/observability/datadog.mdx index fcbdff7ffd19..2436cfcdce36 100644 --- a/website/content/docs/k8s/connect/observability/datadog.mdx +++ b/website/content/docs/k8s/connect/observability/datadog.mdx @@ -395,9 +395,9 @@ This method implements the collection via Openmetrics as that is fully supported #### Use Case -This method of integration is useful for cases where Prometheus enabled scrapes are desired, and further customization of the collected data is desired. +This method of integration is useful for Prometheus-enabled scrapes with further customization of the collected data. -By default, all metrics pulled using this method will scrape Consul metrics using the `/v1/agent/metrics?format=prometheus` API query, and are considered to be **_custom metrics_**. +By default, all metrics pulled using this method scrape Consul metrics using the `/v1/agent/metrics?format=prometheus` API query, and are considered to be custom metrics. Use of this method maps to Datadog as described in [Mapping Prometheus Metrics to Datadog Metrics](https://docs.datadoghq.com/integrations/guide/prometheus-metrics/?tab=latestversion). The following table summarizing how these metrics map to each other. From 2863456521ab579d3b2c8fd1b07fed53c3ff3bfa Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Sat, 17 Feb 2024 00:44:30 +0000 Subject: [PATCH 17/19] backport of commit 96252cc15f80b0818ce43b52b07ad40ed403a954 --- .../datadog.mdx | 0 website/data/docs-nav-data.json | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename website/content/docs/k8s/{connect/observability => deployment-configurations}/datadog.mdx (100%) diff --git a/website/content/docs/k8s/connect/observability/datadog.mdx b/website/content/docs/k8s/deployment-configurations/datadog.mdx similarity index 100% rename from website/content/docs/k8s/connect/observability/datadog.mdx rename to website/content/docs/k8s/deployment-configurations/datadog.mdx diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index ce6653e6ead8..bd93c19f126f 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -1284,6 +1284,10 @@ "path": "k8s/deployment-configurations/vault/wan-federation" } ] + }, + { + "title": "Datadog", + "path": "k8s/deployment-configurations/datadog" } ] }, @@ -1408,10 +1412,6 @@ { "title": "Metrics", "path": "k8s/connect/observability/metrics" - }, - { - "title": "Datadog", - "path": "k8s/connect/observability/datadog" } ] } From 7d6287bbb95a4be5b894f54c3564f50fb4cd5a81 Mon Sep 17 00:00:00 2001 From: natemollica-dev <57850649+natemollica-nm@users.noreply.github.com> Date: Mon, 19 Feb 2024 01:38:02 +0000 Subject: [PATCH 18/19] backport of commit 87e8d65bdc6e62345a3d93f0a3610517f200277f --- website/data/docs-nav-data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index bd93c19f126f..c1603b6bcac3 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -1286,7 +1286,7 @@ ] }, { - "title": "Datadog", + "title": "Datadog metrics", "path": "k8s/deployment-configurations/datadog" } ] From 0ac74adad33b145871fa9b2cdd51c95819154e5e Mon Sep 17 00:00:00 2001 From: natemollica-dev Date: Tue, 20 Feb 2024 01:32:18 +0000 Subject: [PATCH 19/19] backport of commit fc4c8f72bdf1b85e07404fafc2bbb6d56d6c5b92 --- .../k8s/deployment-configurations/datadog.mdx | 62 ++++++++++++++++--- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/website/content/docs/k8s/deployment-configurations/datadog.mdx b/website/content/docs/k8s/deployment-configurations/datadog.mdx index 2436cfcdce36..cc627cb59b61 100644 --- a/website/content/docs/k8s/deployment-configurations/datadog.mdx +++ b/website/content/docs/k8s/deployment-configurations/datadog.mdx @@ -292,20 +292,62 @@ during normal operation beyond that of Consul's available metrics. See the below [table](#additional-integration-checks-performed) for an outline of the features added by the official integration. + + Currently, the annotations configured by the Helm overrides with Consul RPC TLS enabled + assume server and ca certificate secrets are shared with the Datadog agent release namespace and mount the valid tls.crt, tls.key, + and ca.crt secret volumes at the /etc/datadog-agent/conf.d/consul.d/certs path on the Datadog Agent, agent container. + + ### Helm Chart Configuration - + + + + Consul Helm Chart Overrides ```yaml - metrics: - enabled: true - enableAgentMetrics: true - datadog: + global: + tls: enabled: true - namespace: "datadog" + enableAutoEncrypt: true + acls: + manageSystemACLs: true + metrics: + enabled: true + enableAgentMetrics: true + datadog: + enabled: true + namespace: "datadog" ``` - + + Consul `server-statefulset.yaml` annotations + + ```yaml + "ad.datadoghq.com/consul.checks": | + { + "consul": { + "init_config": {}, + "instances": [ + { + "url": "https://consul-server.consul.svc:8501", + "tls_cert": "/etc/datadog-agent/conf.d/consul.d/certs/tls.crt", + "tls_private_key": "/etc/datadog-agent/conf.d/consul.d/certs/tls.key", + "tls_ca_cert": "/etc/datadog-agent/conf.d/consul.d/ca/tls.crt", + "use_prometheus_endpoint": true, + "acl_token": "ENC[k8s_secret@consul/consul-datadog-agent-metrics-acl-token/token]", + "new_leader_checks": true, + "network_latency_checks": true, + "catalog_checks": true, + "auth_type": "basic" + } + ] + } + } + ``` + + + ### Additional Integration Checks Performed @@ -342,6 +384,12 @@ This method implements the collection via Openmetrics as that is fully supported to scrape the agent's metrics API endpoint using either RPC TLS and Consul ACLs as necessary. + + Currently, the annotations configured by the Helm overrides with Consul RPC TLS enabled + assume server and ca certificate secrets are shared with the Datadog agent release namespace and mount the valid tls.crt, tls.key, + and ca.crt secret volumes at the /etc/datadog-agent/conf.d/consul.d/certs path on the Datadog Agent, agent container. + + ### Helm Chart Configuration