Kubernetes: Datadog Constant Tags (#15213)

* adding option for constant tags to be sent to datadog for split environments * removing unnecessary space * Adding tests on envconfig * clean test * adding empty string test for constant tags * format files * remove public class from test class Co-authored-by: Guy Feldman <gfeldman@86labs.com>
airbytehq · Aug 3, 2022 · c5a9c27 · c5a9c27
1 parent fc0bda7
commit c5a9c27
Show file tree

Hide file tree

Showing 8 changed files with 43 additions and 4 deletions.
diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java
@@ -461,6 +461,12 @@ public interface Configs {
    */
   String getDDDogStatsDPort();
 
+  /**
+   * Set constant tags to be attached to all metrics. Useful for distinguishing between environments.
+   * Example: airbyte_instance:dev,k8s-cluster:aws-dev
+   */
+  List<String> getDDConstantTags();
+
   /**
    * Define whether to publish tracking events to Segment or log-only. Airbyte internal use.
    */

diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java
@@ -99,7 +99,7 @@ public class EnvConfigs implements Configs {
   private static final String CONTAINER_ORCHESTRATOR_IMAGE = "CONTAINER_ORCHESTRATOR_IMAGE";
   private static final String DD_AGENT_HOST = "DD_AGENT_HOST";
   private static final String DD_DOGSTATSD_PORT = "DD_DOGSTATSD_PORT";
-
+  public static final String DD_CONSTANT_TAGS = "DD_CONSTANT_TAGS";
   public static final String STATE_STORAGE_S3_BUCKET_NAME = "STATE_STORAGE_S3_BUCKET_NAME";
   public static final String STATE_STORAGE_S3_REGION = "STATE_STORAGE_S3_REGION";
   public static final String STATE_STORAGE_S3_ACCESS_KEY = "STATE_STORAGE_S3_ACCESS_KEY";
@@ -796,6 +796,15 @@ public String getDDDogStatsDPort() {
     return getEnvOrDefault(DD_DOGSTATSD_PORT, "");
   }
 
+  @Override
+  public List<String> getDDConstantTags() {
+    String tagsString = getEnvOrDefault(DD_CONSTANT_TAGS, "");
+    return Splitter.on(",")
+        .splitToStream(tagsString)
+        .filter(s -> !s.trim().isBlank())
+        .collect(Collectors.toList());
+  }
+
   @Override
   public TrackingStrategy getTrackingStrategy() {
     return getEnvOrDefault(TRACKING_STRATEGY, TrackingStrategy.LOGGING, s -> {

diff --git a/airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java b/airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java
@@ -356,6 +356,20 @@ void testPublishMetrics() {
     assertFalse(config.getPublishMetrics());
   }
 
+  @Test
+  @DisplayName("Should parse constant tags")
+  void testDDConstantTags() {
+    assertEquals(List.of(), config.getDDConstantTags());
+
+    envMap.put(EnvConfigs.DD_CONSTANT_TAGS, " ");
+    assertEquals(List.of(), config.getDDConstantTags());
+
+    envMap.put(EnvConfigs.DD_CONSTANT_TAGS, "airbyte_instance:dev,k8s-cluster:eks-dev");
+    List<String> expected = List.of("airbyte_instance:dev", "k8s-cluster:eks-dev");
+    assertEquals(expected, config.getDDConstantTags());
+    assertEquals(2, config.getDDConstantTags().size());
+  }
+
   @Nested
   @DisplayName("CheckJobResourceSettings")
   class CheckJobResourceSettings {

diff --git a/...-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DatadogClientConfiguration.java b/...-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DatadogClientConfiguration.java
@@ -5,6 +5,7 @@
 package io.airbyte.metrics.lib;
 
 import io.airbyte.config.Configs;
+import java.util.List;
 import lombok.AllArgsConstructor;
 
 /**
@@ -17,10 +18,13 @@ public class DatadogClientConfiguration {
   public final String ddPort;
   public final boolean publish;
 
+  public final List<String> constantTags;
+
   public DatadogClientConfiguration(final Configs configs) {
     this.ddAgentHost = configs.getDDAgentHost();
     this.ddPort = configs.getDDDogStatsDPort();
     this.publish = configs.getPublishMetrics();
+    this.constantTags = configs.getDDConstantTags();
   }
 
 }
diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java
@@ -55,6 +55,7 @@ public void initialize(final MetricEmittingApp app, final DatadogClientConfigura
         .prefix(app.getApplicationName())
         .hostname(config.ddAgentHost)
         .port(Integer.parseInt(config.ddPort))
+        .constantTags(config.constantTags.toArray(new String[0]))
         .build();
   }
 

diff --git a/...te-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricSingleton.java b/...te-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricSingleton.java
@@ -46,6 +46,7 @@ public synchronized static void initialize(final MetricEmittingApp app, final Da
         .prefix(app.getApplicationName())
         .hostname(config.ddAgentHost)
         .port(Integer.parseInt(config.ddPort))
+        .constantTags(config.constantTags.toArray(new String[0]))
         .build();
   }
 

diff --git a/...e-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/DogStatsDMetricClientTest.java b/...e-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/DogStatsDMetricClientTest.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.metrics.lib;
 
+import java.util.Collections;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
@@ -17,7 +18,7 @@ class DogStatsDMetricClientTest {
   @BeforeEach
   void setUp() {
     dogStatsDMetricClient = new DogStatsDMetricClient();
-    dogStatsDMetricClient.initialize(MetricEmittingApps.WORKER, new DatadogClientConfiguration("localhost", "1000", false));
+    dogStatsDMetricClient.initialize(MetricEmittingApps.WORKER, new DatadogClientConfiguration("localhost", "1000", false, Collections.emptyList()));
   }
 
   @AfterEach

diff --git a/...etrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/DogStatsDMetricSingletonTest.java b/...etrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/DogStatsDMetricSingletonTest.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.metrics.lib;
 
+import java.util.Collections;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.DisplayName;
@@ -20,7 +21,8 @@ void tearDown() {
   @DisplayName("there should be no exception if we attempt to emit metrics while publish is false")
   void testPublishTrueNoEmitError() {
     Assertions.assertDoesNotThrow(() -> {
-      DogStatsDMetricSingleton.initialize(MetricEmittingApps.WORKER, new DatadogClientConfiguration("localhost", "1000", false));
+      DogStatsDMetricSingleton.initialize(MetricEmittingApps.WORKER,
+          new DatadogClientConfiguration("localhost", "1000", false, Collections.emptyList()));
       DogStatsDMetricSingleton.gauge(OssMetricsRegistry.KUBE_POD_PROCESS_CREATE_TIME_MILLISECS, 1);
     });
   }
@@ -29,7 +31,8 @@ void testPublishTrueNoEmitError() {
   @DisplayName("there should be no exception if we attempt to emit metrics while publish is true")
   void testPublishFalseNoEmitError() {
     Assertions.assertDoesNotThrow(() -> {
-      DogStatsDMetricSingleton.initialize(MetricEmittingApps.WORKER, new DatadogClientConfiguration("localhost", "1000", true));
+      DogStatsDMetricSingleton.initialize(MetricEmittingApps.WORKER,
+          new DatadogClientConfiguration("localhost", "1000", true, Collections.emptyList()));
       DogStatsDMetricSingleton.gauge(OssMetricsRegistry.KUBE_POD_PROCESS_CREATE_TIME_MILLISECS, 1);
     });
   }