otlp: datadog otlp acceptance test updates for telemetry-collector (g…

…rpc => http prefix) | staged otlp acceptance test
hashicorp · Feb 24, 2024 · a7d0c7e · a7d0c7e
1 parent 0c80934
commit a7d0c7e
Show file tree

Hide file tree

Showing 8 changed files with 183 additions and 17 deletions.
diff --git a/acceptance/framework/datadog/datadog.go b/acceptance/framework/datadog/datadog.go
@@ -22,6 +22,7 @@ import (
 const (
 	MetricsListQuery           = "search"
 	MetricTimeSeriesQuery      = "time-series"
+	MetricTimeSeriesOTLPQuery  = "time-series-otlp"
 	releaseLabel               = "app.kubernetes.io/name"
 	DatadogOperatorReleaseName = "datadog-operator"
 	DefaultHelmChartVersion    = "1.4.0"
@@ -55,7 +56,7 @@ func (d *DatadogCluster) releaseLabelSelector() string {
 	return fmt.Sprintf("%s=%s", releaseLabel, d.releaseName)
 }
 
-// DatadogClient returns datadog client
+// DatadogClient returns datadog client.
 func (d *DatadogCluster) DatadogClient(*testing.T) *DatadogClient { return d.datadogClient }
 
 // NewDatadogClient initializes and returns a DataDog client using the API key and Application key from environment variables.

diff --git a/acceptance/framework/datadog/helpers.go b/acceptance/framework/datadog/helpers.go
@@ -35,6 +35,11 @@ func ApiWithRetry(t *testing.T, apiClient *DatadogClient, endpoint, testTags, qu
 			fullQueryString := fmt.Sprintf("avg:%s{%s}", query, testTags)
 			queryResponse, fullResponse, err = api.QueryMetrics(apiClient.Ctx, time.Now().Add(-1*time.Minute).Unix(), time.Now().Unix(), fullQueryString)
 			response.QueryResponse = &queryResponse
+		case MetricTimeSeriesOTLPQuery:
+			var queryResponse datadogV1.MetricsQueryResponse
+			fullQueryString := fmt.Sprintf("sum:%s{%s}", query, testTags)
+			queryResponse, fullResponse, err = api.QueryMetrics(apiClient.Ctx, time.Now().Add(-1*time.Minute).Unix(), time.Now().Unix(), fullQueryString)
+			response.QueryResponse = &queryResponse
 		default:
 			var searchResponse datadogV1.MetricSearchResponse
 			searchResponse, fullResponse, err = api.ListMetrics(apiClient.Ctx, query)
@@ -46,13 +51,15 @@ func ApiWithRetry(t *testing.T, apiClient *DatadogClient, endpoint, testTags, qu
 
 		// Log the error and response details
 		content, _ := json.MarshalIndent(response.SearchResponse, "", "    ")
-		if endpoint == "time-series" {
+		sentQuery := query
+		if endpoint == MetricTimeSeriesQuery || endpoint == MetricTimeSeriesOTLPQuery {
 			content, _ = json.MarshalIndent(response.QueryResponse, "", "    ")
+			sentQuery = response.QueryResponse.GetQuery()
 		}
 		if err != nil {
-			logger.Logf(t, "Attempt %d of %d: Error received when calling Datadog API MetricsApi.ListMetrics (/v1/search) | query=%s | error=%v", attempt+1, maxRetries, query, err)
+			logger.Logf(t, "Attempt %d of %d: Error received when calling Datadog API | query=%s | error=%v", attempt+1, maxRetries, sentQuery, err)
 		} else {
-			logger.Logf(t, "Attempt %d of %d: No match received when calling Datadog API MetricsApi.ListMetrics (/v1/search) | query=%s", attempt+1, maxRetries, query)
+			logger.Logf(t, "Attempt %d of %d: No match received when calling Datadog API | query=%s", attempt+1, maxRetries, sentQuery)
 		}
 		logger.Logf(t, "Response: %v", string(content))
 
@@ -65,10 +72,10 @@ func ApiWithRetry(t *testing.T, apiClient *DatadogClient, endpoint, testTags, qu
 	return response, fullResponse, err
 }
 
-// getBackoffDuration calculates the time to wait before the next retry attempt with an exponential backoff and jitter
+// getBackoffDuration calculates the time to wait before the next retry attempt with an exponential backoff and jitter.
 func getBackoffDuration(attempt int) time.Duration {
 	// Exponential backoff factor
-	backoff := math.Pow(7, float64(attempt)) * 100 // Base delay in milliseconds
+	backoff := math.Pow(5, float64(attempt)) * 100 // Base delay in milliseconds
 	jitter := rand.Float64() * 100                 // Add jitter up to 100ms
 	return time.Duration(backoff+jitter) * time.Millisecond
 }

diff --git a/acceptance/tests/datadog/datadog_test.go b/acceptance/tests/datadog/datadog_test.go
@@ -13,16 +13,17 @@ import (
 )
 
 const (
-	maxDatadogAPIRetryAttempts   = 20
+	maxDatadogAPIRetryAttempts   = 5
 	consulDogstatsDMetricQuery   = "consul.memberlist.gossip.50percentile"
 	consulIntegrationMetricQuery = "consul.memberlist.gossip.quantile"
+	consulOTLPMetricQuery        = `otelcol_process_runtime_heap_alloc_bytes`
 )
 
 // TestDatadogDogstatsDUnixDomainSocket
 // Acceptance test to verify e2e metrics configuration works as expected
 // with live datadog API using histogram formatted metric
 //
-// Method: DogstatsD + Unix Domain Socket
+// Method: DogstatsD + Unix Domain Socket.
 func TestDatadogDogstatsDUnixDomainSocket(t *testing.T) {
 	env := suite.Environment()
 	cfg := suite.Config()
@@ -84,7 +85,7 @@ func TestDatadogDogstatsDUnixDomainSocket(t *testing.T) {
 // Acceptance test to verify e2e metrics configuration works as expected
 // with live datadog API using histogram formatted metric
 //
-// Method: DogstatsD + UDP to Kube Service DNS name on port 8125
+// Method: DogstatsD + UDP to Kube Service DNS name on port 8125.
 func TestDatadogDogstatsDUDP(t *testing.T) {
 	env := suite.Environment()
 	cfg := suite.Config()
@@ -120,7 +121,7 @@ func TestDatadogDogstatsDUDP(t *testing.T) {
 	consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName)
 	consulCluster.Create(t)
 
-	// Deploy Datadog Agent via Datadog Operator and apply dogstatsd overlay
+	// Deploy Datadog Agent via Datadog Operator and apply dogstatsd overlay.
 	datadogNamespace := helmValues["global.metrics.datadog.namespace"]
 	logger.Log(t, fmt.Sprintf("deploying datadog-operator via helm | namespace: %s | release-name: %s", datadogNamespace, datadogOperatorRelease))
 	datadogCluster := datadog.NewDatadogCluster(t, ctx, cfg, datadogOperatorRelease, datadogNamespace, datadogOperatorHelmValues)
@@ -148,7 +149,7 @@ func TestDatadogDogstatsDUDP(t *testing.T) {
 // Acceptance test to verify e2e metrics configuration works as expected
 // with live datadog API using histogram formatted metric
 //
-// Method: Consul Integrated Datadog Checks
+// Method: Consul Integrated Datadog Checks.
 func TestDatadogConsulChecks(t *testing.T) {
 	env := suite.Environment()
 	cfg := suite.Config()
@@ -180,7 +181,7 @@ func TestDatadogConsulChecks(t *testing.T) {
 	consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName)
 	consulCluster.Create(t)
 
-	// Deploy Datadog Agent via Datadog Operator and apply dogstatsd overlay
+	// Deploy Datadog Agent via Datadog Operator and apply dogstatsd overlay.
 	datadogNamespace := helmValues["global.metrics.datadog.namespace"]
 	logger.Log(t, fmt.Sprintf("deploying datadog-operator via helm | namespace: %s | release-name: %s", datadogNamespace, datadogOperatorRelease))
 	datadogCluster := datadog.NewDatadogCluster(t, ctx, cfg, datadogOperatorRelease, datadogNamespace, datadogOperatorHelmValues)
@@ -208,7 +209,7 @@ func TestDatadogConsulChecks(t *testing.T) {
 // Acceptance test to verify e2e metrics configuration works as expected
 // with live datadog API using histogram formatted metric
 //
-// Method: Datadog Openmetrics Prometheus Metrics Collection
+// Method: Datadog Openmetrics Prometheus Metrics Collection.
 func TestDatadogOpenmetrics(t *testing.T) {
 	env := suite.Environment()
 	cfg := suite.Config()
@@ -264,3 +265,127 @@ func TestDatadogOpenmetrics(t *testing.T) {
 	logger.Logf(t, "Response: %v", string(content))
 	require.Contains(t, string(content), consulOpenmetricsQuery)
 }
+
+// TestDatadogOTLPCollection
+// Acceptance test to verify e2e metrics configuration works as expected
+// with live datadog API using histogram formatted metric
+//
+// Method: Datadog otlp metrics collection via consul-telemetry collector using dd-agent gRPC receiver.
+func TestDatadogOTLPCollection(t *testing.T) {
+	env := suite.Environment()
+	cfg := suite.Config()
+	ctx := env.DefaultContext(t)
+	// ns := ctx.KubectlOptions(t).Namespace
+
+	helmValues := map[string]string{
+		"global.datacenter":                    "dc1",
+		"global.metrics.enabled":               "true",
+		"global.metrics.enableAgentMetrics":    "true",
+		"global.metrics.disableAgentHostName":  "true",
+		"global.metrics.enableHostMetrics":     "true",
+		"global.metrics.datadog.enabled":       "true",
+		"global.metrics.datadog.namespace":     "datadog",
+		"global.metrics.datadog.otlp.enabled":  "true",
+		"global.metrics.datadog.otlp.protocol": "http",
+		"telemetryCollector.enabled":           "true",
+	}
+
+	datadogOperatorHelmValues := map[string]string{
+		"replicaCount":     "1",
+		"image.tag":        datadog.DefaultHelmChartVersion,
+		"image.repository": "gcr.io/datadoghq/operator",
+	}
+
+	releaseName := helpers.RandomName()
+	datadogOperatorRelease := datadog.DatadogOperatorReleaseName
+
+	acceptanceTestingTags := `service:consul-telemetry-collector`
+	// Install the consul cluster in the default kubernetes ctx.
+	consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName)
+	consulCluster.Create(t)
+
+	// Deploy Datadog Agent via Datadog Operator and apply dogstatsd overlay
+	datadogNamespace := helmValues["global.metrics.datadog.namespace"]
+	logger.Log(t, fmt.Sprintf("deploying datadog-operator via helm | namespace: %s | release-name: %s", datadogNamespace, datadogOperatorRelease))
+	datadogCluster := datadog.NewDatadogCluster(t, ctx, cfg, datadogOperatorRelease, datadogNamespace, datadogOperatorHelmValues)
+	datadogCluster.Create(t)
+
+	logger.Log(t, fmt.Sprintf("applying datadog otlp HTTP endpoint collector patch to datadog-agent | namespace: %s", datadogNamespace))
+	k8s.DeployKustomize(t, ctx.KubectlOptionsForNamespace(datadogNamespace), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/datadog-otlp")
+	k8s.WaitForAllPodsToBeReady(t, ctx.KubernetesClient(t), datadogNamespace, "agent.datadoghq.com/component=agent")
+
+	datadogAPIClient := datadogCluster.DatadogClient(t)
+	response, fullResponse, err := datadog.ApiWithRetry(t, datadogAPIClient, datadog.MetricTimeSeriesOTLPQuery, acceptanceTestingTags, consulOTLPMetricQuery, maxDatadogAPIRetryAttempts)
+	if err != nil {
+		content, _ := json.MarshalIndent(response.QueryResponse, "", "   ")
+		fullContent, _ := json.MarshalIndent(fullResponse, "", "    ")
+		logger.Logf(t, "Error when querying /v1/query endpoint for %s: %v", consulOTLPMetricQuery, err)
+		logger.Logf(t, "Response: %v", string(content))
+		logger.Logf(t, "Full Response: %v", string(fullContent))
+	}
+	content, _ := json.MarshalIndent(response.QueryResponse, "", "   ")
+	logger.Logf(t, "Response: %v", string(content))
+	require.Contains(t, string(content), consulOTLPMetricQuery)
+}
+
+// TestDatadogOTLPgRPCCollection (pending: https://github.com/hashicorp/consul-telemetry-collector/pull/124 merge with consul-telemetry-collector)
+// Acceptance test to verify e2e metrics configuration works as expected
+// with live datadog API using histogram formatted metric
+//
+// Method: Datadog otlp metrics collection via consul-telemetry collector using dd-agent gRPC receiver.
+//func TestDatadogOTLPgRPCCollection(t *testing.T) {
+//	env := suite.Environment()
+//	cfg := suite.Config()
+//	ctx := env.DefaultContext(t)
+//	// ns := ctx.KubectlOptions(t).Namespace
+//
+//	helmValues := map[string]string{
+//		"global.datacenter":                    "dc1",
+//		"global.metrics.enabled":               "true",
+//		"global.metrics.enableAgentMetrics":    "true",
+//		"global.metrics.disableAgentHostName":  "true",
+//		"global.metrics.enableHostMetrics":     "true",
+//		"global.metrics.datadog.enabled":       "true",
+//		"global.metrics.datadog.namespace":     "datadog",
+//		"global.metrics.datadog.otlp.enabled":  "true",
+//		"global.metrics.datadog.otlp.protocol": "grpc",
+//		"telemetryCollector.enabled":           "true",
+//	}
+//
+//	datadogOperatorHelmValues := map[string]string{
+//		"replicaCount":     "1",
+//		"image.tag":        datadog.DefaultHelmChartVersion,
+//		"image.repository": "gcr.io/datadoghq/operator",
+//	}
+//
+//	releaseName := helpers.RandomName()
+//	datadogOperatorRelease := datadog.DatadogOperatorReleaseName
+//
+//	acceptanceTestingTags := `service:consul-telemetry-collector`
+//	// Install the consul cluster in the default kubernetes ctx.
+//	consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName)
+//	consulCluster.Create(t)
+//
+//	// Deploy Datadog Agent via Datadog Operator and apply dogstatsd overlay
+//	datadogNamespace := helmValues["global.metrics.datadog.namespace"]
+//	logger.Log(t, fmt.Sprintf("deploying datadog-operator via helm | namespace: %s | release-name: %s", datadogNamespace, datadogOperatorRelease))
+//	datadogCluster := datadog.NewDatadogCluster(t, ctx, cfg, datadogOperatorRelease, datadogNamespace, datadogOperatorHelmValues)
+//	datadogCluster.Create(t)
+//
+//	logger.Log(t, fmt.Sprintf("applying datadog otlp gRPC endpoint collector patch to datadog-agent | namespace: %s", datadogNamespace))
+//	k8s.DeployKustomize(t, ctx.KubectlOptionsForNamespace(datadogNamespace), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/datadog-otlp-grpc")
+//	k8s.WaitForAllPodsToBeReady(t, ctx.KubernetesClient(t), datadogNamespace, "agent.datadoghq.com/component=agent")
+//
+//	datadogAPIClient := datadogCluster.DatadogClient(t)
+//	response, fullResponse, err := datadog.ApiWithRetry(t, datadogAPIClient, datadog.MetricTimeSeriesOTLPQuery, acceptanceTestingTags, consulOTLPMetricQuery, maxDatadogAPIRetryAttempts)
+//	if err != nil {
+//		content, _ := json.MarshalIndent(response.QueryResponse, "", "   ")
+//		fullContent, _ := json.MarshalIndent(fullResponse, "", "    ")
+//		logger.Logf(t, "Error when querying /v1/query endpoint for %s: %v", consulOTLPMetricQuery, err)
+//		logger.Logf(t, "Response: %v", string(content))
+//		logger.Logf(t, "Full Response: %v", string(fullContent))
+//	}
+//	content, _ := json.MarshalIndent(response.QueryResponse, "", "   ")
+//	logger.Logf(t, "Response: %v", string(content))
+//	require.Contains(t, string(content), consulOTLPMetricQuery)
+//}
diff --git a/acceptance/tests/fixtures/cases/datadog-otlp-grpc/kustomization.yaml b/acceptance/tests/fixtures/cases/datadog-otlp-grpc/kustomization.yaml
@@ -0,0 +1,8 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../bases/datadog
+patches:
+  - path: patch.yaml
diff --git a/acceptance/tests/fixtures/cases/datadog-otlp-grpc/patch.yaml b/acceptance/tests/fixtures/cases/datadog-otlp-grpc/patch.yaml
@@ -0,0 +1,16 @@
+apiVersion: datadoghq.com/v2alpha1
+kind: DatadogAgent
+metadata:
+  name: datadog
+spec:
+  features:
+    # Sets: DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT:  0.0.0.0:4317                                                                                                                                              │
+    #       DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT:  0.0.0.0:4318
+    otlp:
+      receiver:
+        protocols:
+          # Set to "0.0.0.0" as per the below reference docs
+          # ref: https://docs.datadoghq.com/opentelemetry/otlp_ingest_in_the_agent/?tab=host#enabling-otlp-ingestion-on-the-datadog-agent
+          grpc:
+            enabled: true
+            endpoint: "0.0.0.0:4317"
diff --git a/acceptance/tests/fixtures/cases/datadog-otlp/kustomization.yaml b/acceptance/tests/fixtures/cases/datadog-otlp/kustomization.yaml
@@ -0,0 +1,8 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../bases/datadog
+patches:
+  - path: patch.yaml
diff --git a/acceptance/tests/fixtures/cases/datadog-otlp/patch.yaml b/acceptance/tests/fixtures/cases/datadog-otlp/patch.yaml
@@ -4,12 +4,13 @@ metadata:
   name: datadog
 spec:
   features:
+    # Sets: DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT:  0.0.0.0:4317                                                                                                                                              │
+    #       DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT:  0.0.0.0:4318
     otlp:
       receiver:
         protocols:
-          grpc:
-            enabled: true
-            endpoint: "0.0.0.0:4317"
+          # Set to "0.0.0.0" as per the below reference docs
+          # ref: https://docs.datadoghq.com/opentelemetry/otlp_ingest_in_the_agent/?tab=host#enabling-otlp-ingestion-on-the-datadog-agent
           http:
             enabled: true
             endpoint: "0.0.0.0:4318"
diff --git a/charts/consul/templates/telemetry-collector-deployment.yaml b/charts/consul/templates/telemetry-collector-deployment.yaml
@@ -260,7 +260,7 @@ spec:
               value: "http://$(HOST_IP):4318"
             {{- else if eq (.Values.global.metrics.datadog.otlp.protocol | lower) "grpc" }}
             - name: CO_OTEL_HTTP_ENDPOINT
-              value: "grpc://$(HOST_IP):4317"
+              value: "http://$(HOST_IP):4317"
             - name: OTEL_EXPORTER_OTLP_ENDPOINT
               value: "http://$(HOST_IP):4317"
             {{- end }}