Add env var substitution support to file configuration

sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationFactory.java

@@ -34,6 +34,9 @@ private ConfigurationFactory() {}
    * Parse the {@code inputStream} YAML to {@link OpenTelemetryConfiguration} and interpret the
    * model to create {@link OpenTelemetrySdk} instance corresponding to the configuration.
    *
+   * <p>Before parsing, environment variable substitution is performed as described in {@link
+   * ConfigurationReader.EnvSubstitutionConstructor}.
+   *
    * @param inputStream the configuration YAML
    * @return the {@link OpenTelemetrySdk}
    */

sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReader.java

@@ -10,23 +10,120 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.opentelemetry.sdk.extension.incubator.fileconfig.internal.model.OpenTelemetryConfiguration;
 import java.io.InputStream;
+import java.util.Map;
+import java.util.regex.MatchResult;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.snakeyaml.engine.v2.api.Load;
 import org.snakeyaml.engine.v2.api.LoadSettings;
+import org.snakeyaml.engine.v2.constructor.StandardConstructor;
+import org.snakeyaml.engine.v2.nodes.MappingNode;
+import org.yaml.snakeyaml.Yaml;
 
 final class ConfigurationReader {
 
-  private static final ObjectMapper MAPPER =
-      new ObjectMapper()
-          // Create empty object instances for keys which are present but have null values
-          .setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY));
+  private static final Pattern ENV_VARIABLE_REFERENCE =
+      Pattern.compile("\\$\\{env:([a-zA-Z_]+[a-zA-Z0-9_]*)}");
+
+  private static final ObjectMapper MAPPER;
+
+  static {
+    MAPPER =
+        new ObjectMapper()
+            // Create empty object instances for keys which are present but have null values
+            .setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY));
+    // Boxed primitives which are present but have null values should be set to null, rather than
+    // empty instances
+    MAPPER.configOverride(String.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
+    MAPPER.configOverride(Integer.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
+    MAPPER.configOverride(Double.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
+    MAPPER.configOverride(Boolean.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
+  }
 
   private ConfigurationReader() {}
 
-  /** Parse the {@code configuration} YAML and return the {@link OpenTelemetryConfiguration}. */
+  /**
+   * Parse the {@code configuration} YAML and return the {@link OpenTelemetryConfiguration}.
+   *
+   * <p>Before parsing, environment variable substitution is performed as described in {@link
+   * EnvSubstitutionConstructor}.
+   */
   static OpenTelemetryConfiguration parse(InputStream configuration) {
-    LoadSettings settings = LoadSettings.builder().build();
-    Load yaml = new Load(settings);
-    Object yamlObj = yaml.loadFromInputStream(configuration);
+    return parse(configuration, System.getenv());
+  }
+
+  // Visible for testing
+  static OpenTelemetryConfiguration parse(
+      InputStream configuration, Map<String, String> environmentVariables) {
+    Object yamlObj = loadYaml(configuration, environmentVariables);
     return MAPPER.convertValue(yamlObj, OpenTelemetryConfiguration.class);
   }
+
+  static Object loadYaml(InputStream inputStream, Map<String, String> environmentVariables) {
+    LoadSettings settings = LoadSettings.builder().build();
+    Load yaml = new Load(settings, new EnvSubstitutionConstructor(settings, environmentVariables));
+    return yaml.loadFromInputStream(inputStream);
+  }
+
+  /**
+   * {@link StandardConstructor} which substitutes environment variables.
+   *
+   * <p>Environment variables follow the syntax {@code ${env:VARIABLE}}, where {@code VARIABLE} is
+   * an environment variable matching the regular expression {@code [a-zA-Z_]+[a-zA-Z0-9_]*}.
+   *
+   * <p>Environment variable substitution only takes place on scalar values of maps. References to
+   * environment variables in keys or sets are ignored.
+   *
+   * <p>If a referenced environment variable is not defined, it is replaced with {@code ""}.
+   */
+  static final class EnvSubstitutionConstructor extends StandardConstructor {
+
+    // Yaml is not thread safe but this instance is always used on the same thread
+    private final Yaml yaml = new Yaml();
+    private final Map<String, String> environmentVariables;
+
+    private EnvSubstitutionConstructor(
+        LoadSettings loadSettings, Map<String, String> environmentVariables) {
+      super(loadSettings);
+      this.environmentVariables = environmentVariables;
+    }
+
+    @Override
+    protected Map<Object, Object> constructMapping(MappingNode node) {
+      // First call the super to construct mapping from MappingNode as usual
+      Map<Object, Object> result = super.constructMapping(node);
+
+      // Iterate through the map entries, and:
+      // 1. Identify entries which are scalar strings eligible for environment variable substitution
+      // 2. Apply environment variable substitution
+      // 3. Re-parse substituted value so it has correct type (i.e. yaml.load(newVal))
+      for (Map.Entry<Object, Object> entry : result.entrySet()) {
+        Object value = entry.getValue();
+        if (!(value instanceof String)) {
+          continue;
+        }
+
+        String val = (String) value;
+        Matcher matcher = ENV_VARIABLE_REFERENCE.matcher(val);
+        if (!matcher.find()) {
+          continue;
+        }
+
+        int offset = 0;
+        StringBuilder newVal = new StringBuilder();
+        do {
+          MatchResult matchResult = matcher.toMatchResult();
+          String replacement = environmentVariables.getOrDefault(matcher.group(1), "");
+          newVal.append(val, offset, matchResult.start()).append(replacement);
+          offset = matchResult.end();
+        } while (matcher.find());
+        if (offset != val.length()) {
+          newVal.append(val, offset, val.length());
+        }
+        entry.setValue(yaml.load(newVal.toString()));
+      }
+
+      return result;
+    }
+  }
 }

sdk-extensions/incubator/src/test/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReaderTest.java

@@ -49,14 +49,21 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.AbstractMap;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nullable;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 class ConfigurationReaderTest {
 
   @Test
-  void read_KitchenSinkExampleFile() throws IOException {
+  void parse_KitchenSinkExampleFile() throws IOException {
     OpenTelemetryConfiguration expected = new OpenTelemetryConfiguration();
 
     expected.withFileFormat("0.1");
@@ -283,7 +290,7 @@ void read_KitchenSinkExampleFile() throws IOException {
   }
 
   @Test
-  void nullValuesParsedToEmptyObjects() {
+  void parse_nullValuesParsedToEmptyObjects() {
     String objectPlaceholderString =
         "file_format: \"0.1\"\n"
             + "tracer_provider:\n"
@@ -337,4 +344,139 @@ void nullValuesParsedToEmptyObjects() {
 
     assertThat(objectPlaceholderModel).isEqualTo(noObjectPlaceholderModel);
   }
+
+  @Test
+  void parse_nullBoxedPrimitivesParsedToNull() {
+    String yaml =
+        "file_format:\n" // String
+            + "disabled:\n" // Boolean
+            + "attribute_limits:\n"
+            + "  attribute_value_length_limit:\n" // Integer
+            + "tracer_provider:\n"
+            + "  sampler:\n"
+            + "    trace_id_ratio_based:\n"
+            + "      ratio:\n"; // Double
+
+    OpenTelemetryConfiguration model =
+        ConfigurationReader.parse(new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)));
+
+    assertThat(model.getFileFormat()).isNull();
+    assertThat(model.getDisabled()).isNull();
+    assertThat(model.getAttributeLimits().getAttributeValueLengthLimit()).isNull();
+    assertThat(model.getTracerProvider().getSampler().getTraceIdRatioBased().getRatio()).isNull();
+
+    assertThat(model)
+        .isEqualTo(
+            new OpenTelemetryConfiguration()
+                .withAttributeLimits(new AttributeLimits())
+                .withTracerProvider(
+                    new TracerProvider()
+                        .withSampler(
+                            new Sampler().withTraceIdRatioBased(new TraceIdRatioBased()))));
+  }
+
+  @ParameterizedTest
+  @MethodSource("envVarSubstitutionArgs")
+  void envSubstituteAndLoadYaml(String rawYaml, Object expectedYamlResult) {
+    Map<String, String> environmentVariables = new HashMap<>();
+    environmentVariables.put("STR_1", "value1");
+    environmentVariables.put("STR_2", "value2");
+    environmentVariables.put("BOOL", "true");
+    environmentVariables.put("INT", "1");
+    environmentVariables.put("FLOAT", "1.1");
+
+    Object yaml =
+        ConfigurationReader.loadYaml(
+            new ByteArrayInputStream(rawYaml.getBytes(StandardCharsets.UTF_8)),
+            environmentVariables);
+    assertThat(yaml).isEqualTo(expectedYamlResult);
+  }
+
+  @SuppressWarnings("unchecked")
+  private static java.util.stream.Stream<Arguments> envVarSubstitutionArgs() {
+    return java.util.stream.Stream.of(
+        // Simple cases
+        Arguments.of("key1: ${env:STR_1}\n", mapOf(entry("key1", "value1"))),
+        Arguments.of("key1: ${env:BOOL}\n", mapOf(entry("key1", true))),
+        Arguments.of("key1: ${env:INT}\n", mapOf(entry("key1", 1))),
+        Arguments.of("key1: ${env:FLOAT}\n", mapOf(entry("key1", 1.1))),
+        Arguments.of(
+            "key1: ${env:STR_1}\n" + "key2: value2\n",
+            mapOf(entry("key1", "value1"), entry("key2", "value2"))),
+        Arguments.of(
+            "key1: ${env:STR_1} value1\n" + "key2: value2\n",
+            mapOf(entry("key1", "value1 value1"), entry("key2", "value2"))),
+        // Multiple environment variables referenced
+        Arguments.of("key1: ${env:STR_1}${env:STR_2}\n", mapOf(entry("key1", "value1value2"))),
+        Arguments.of("key1: ${env:STR_1} ${env:STR_2}\n", mapOf(entry("key1", "value1 value2"))),
+        // Undefined environment variable
+        Arguments.of("key1: ${env:STR_3}\n", mapOf(entry("key1", null))),
+        Arguments.of("key1: ${env:STR_1} ${env:STR_3}\n", mapOf(entry("key1", "value1"))),
+        // Environment variable keys must match pattern: [a-zA-Z_]+[a-zA-Z0-9_]*
+        Arguments.of("key1: ${env:VAR&}\n", mapOf(entry("key1", "${env:VAR&}"))),
+        // Environment variable substitution only takes place in scalar values of maps
+        Arguments.of("${env:STR_1}: value1\n", mapOf(entry("${env:STR_1}", "value1"))),
+        Arguments.of(
+            "key1:\n  ${env:STR_1}: value1\n",
+            mapOf(entry("key1", mapOf(entry("${env:STR_1}", "value1"))))),
+        Arguments.of(
+            "key1:\n - ${env:STR_1}\n",
+            mapOf(entry("key1", Collections.singletonList("${env:STR_1}")))));
+  }
+
+  private static <K, V> Map.Entry<K, V> entry(K key, @Nullable V value) {
+    return new AbstractMap.SimpleEntry<>(key, value);
+  }
+
+  @SuppressWarnings("unchecked")
+  private static Map<String, Object> mapOf(Map.Entry<String, ?>... entries) {
+    Map<String, Object> result = new HashMap<>();
+    for (Map.Entry<String, ?> entry : entries) {
+      result.put(entry.getKey(), entry.getValue());
+    }
+    return result;
+  }
+
+  @Test
+  void read_WithEnvironmentVariables() {
+    String yaml =
+        "file_format: \"0.1\"\n"
+            + "tracer_provider:\n"
+            + "  processors:\n"
+            + "    - batch:\n"
+            + "        exporter:\n"
+            + "          otlp:\n"
+            + "            endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT}\n"
+            + "    - batch:\n"
+            + "        exporter:\n"
+            + "          otlp:\n"
+            + "            endpoint: \"${env:UNSET_ENV_VAR}\"\n";
+    Map<String, String> envVars = new HashMap<>();
+    envVars.put("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4317");
+    OpenTelemetryConfiguration model =
+        ConfigurationReader.parse(
+            new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)), envVars);
+    assertThat(model)
+        .isEqualTo(
+            new OpenTelemetryConfiguration()
+                .withFileFormat("0.1")
+                .withTracerProvider(
+                    new TracerProvider()
+                        .withProcessors(
+                            Arrays.asList(
+                                new SpanProcessor()
+                                    .withBatch(
+                                        new BatchSpanProcessor()
+                                            .withExporter(
+                                                new SpanExporter()
+                                                    .withOtlp(
+                                                        new Otlp()
+                                                            .withEndpoint(
+                                                                "http://collector:4317")))),
+                                new SpanProcessor()
+                                    .withBatch(
+                                        new BatchSpanProcessor()
+                                            .withExporter(
+                                                new SpanExporter().withOtlp(new Otlp())))))));
+  }
 }