Add env var substitution support to file configuration

sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationFactory.java

@@ -14,6 +14,7 @@
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.logging.Logger;
 
 /**
@@ -34,6 +35,9 @@ private ConfigurationFactory() {}
    * Parse the {@code inputStream} YAML to {@link OpenTelemetryConfiguration} and interpret the
    * model to create {@link OpenTelemetrySdk} instance corresponding to the configuration.
    *
+   * <p>Before parsing, environment variable substitution is performed as described in {@link
+   * ConfigurationReader#substituteEnvVariables(InputStream, Map)}.
+   *
    * @param inputStream the configuration YAML
    * @return the {@link OpenTelemetrySdk}
    */

sdk-extensions/incubator/src/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReader.java

@@ -8,25 +8,104 @@
 import com.fasterxml.jackson.annotation.JsonSetter;
 import com.fasterxml.jackson.annotation.Nulls;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import io.opentelemetry.sdk.autoconfigure.spi.ConfigurationException;
 import io.opentelemetry.sdk.extension.incubator.fileconfig.internal.model.OpenTelemetryConfiguration;
+import java.io.BufferedReader;
+import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.regex.MatchResult;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.snakeyaml.engine.v2.api.Load;
 import org.snakeyaml.engine.v2.api.LoadSettings;
 
 final class ConfigurationReader {
 
-  private static final ObjectMapper MAPPER =
-      new ObjectMapper()
-          // Create empty object instances for keys which are present but have null values
-          .setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY));
+  private static final Pattern ENV_VARIABLE_REFERENCE =
+      Pattern.compile("\\$\\{env:([a-zA-Z_]+[a-zA-Z0-9_]*)}");
+
+  private static final ObjectMapper MAPPER;
+
+  static {
+    MAPPER =
+        new ObjectMapper()
+            // Create empty object instances for keys which are present but have null values
+            .setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY));
+    // Boxed primitives which are present but have null values should be set to null, rather than
+    // empty instances
+    MAPPER.configOverride(String.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
+    MAPPER.configOverride(Integer.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
+    MAPPER.configOverride(Double.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
+    MAPPER.configOverride(Boolean.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
+  }
 
   private ConfigurationReader() {}
 
-  /** Parse the {@code configuration} YAML and return the {@link OpenTelemetryConfiguration}. */
+  /**
+   * Parse the {@code configuration} YAML and return the {@link OpenTelemetryConfiguration}.
+   *
+   * <p>Before parsing, environment variable substitution is performed as described in {@link
+   * #substituteEnvVariables(InputStream, Map)}.
+   */
   static OpenTelemetryConfiguration parse(InputStream configuration) {
+    return parse(configuration, System.getenv());
+  }
+
+  // Visible for testing
+  static OpenTelemetryConfiguration parse(
+      InputStream configuration, Map<String, String> environmentVariables) {
+    Object yamlObj = loadYaml(configuration, environmentVariables);
+    return MAPPER.convertValue(yamlObj, OpenTelemetryConfiguration.class);
+  }
+
+  static Object loadYaml(InputStream inputStream, Map<String, String> environmentVariables) {
     LoadSettings settings = LoadSettings.builder().build();
     Load yaml = new Load(settings);
-    Object yamlObj = yaml.loadFromInputStream(configuration);
-    return MAPPER.convertValue(yamlObj, OpenTelemetryConfiguration.class);
+    String withEnvironmentVariablesSubstituted =
+        substituteEnvVariables(inputStream, environmentVariables);
+    return yaml.loadFromString(withEnvironmentVariablesSubstituted);
+  }
+
+  /**
+   * Read the input and substitute any environment variables.
+   *
+   * <p>Environment variables follow the syntax {@code ${env:VARIABLE}}, where {@code VARIABLE} is
+   * an environment variable matching the regular expression {@code [a-zA-Z_]+[a-zA-Z0-9_]*}.
+   *
+   * <p>If a referenced environment variable is not defined, it is replaced with {@code ""}.
+   *
+   * @return the string contents of the {@code inputStream} with environment variables substituted
+   */
+  static String substituteEnvVariables(
+      InputStream inputStream, Map<String, String> environmentVariables) {
+    try (BufferedReader reader =
+        new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
+      StringBuilder stringBuilder = new StringBuilder();
+      String line;
+      while ((line = reader.readLine()) != null) {
+        Matcher matcher = ENV_VARIABLE_REFERENCE.matcher(line);
+        if (matcher.find()) {
+          int offset = 0;
+          StringBuilder newLine = new StringBuilder();
+          do {
+            MatchResult matchResult = matcher.toMatchResult();
+            String replacement = environmentVariables.getOrDefault(matcher.group(1), "");
+            newLine.append(line, offset, matchResult.start()).append(replacement);
+            offset = matchResult.end();
+          } while (matcher.find());
+          if (offset != line.length()) {
+            newLine.append(line, offset, line.length());
+          }
+          line = newLine.toString();
+        }
+        stringBuilder.append(line).append(System.lineSeparator());
+      }
+      return stringBuilder.toString();
+    } catch (IOException e) {
+      throw new ConfigurationException("Error reading input stream", e);
+    }
   }
 }

sdk-extensions/incubator/src/test/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReaderTest.java

@@ -49,14 +49,21 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.AbstractMap;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nullable;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 class ConfigurationReaderTest {
 
   @Test
-  void read_KitchenSinkExampleFile() throws IOException {
+  void parse_KitchenSinkExampleFile() throws IOException {
     OpenTelemetryConfiguration expected = new OpenTelemetryConfiguration();
 
     expected.withFileFormat("0.1");
@@ -283,7 +290,7 @@ void read_KitchenSinkExampleFile() throws IOException {
   }
 
   @Test
-  void nullValuesParsedToEmptyObjects() {
+  void parse_nullValuesParsedToEmptyObjects() {
     String objectPlaceholderString =
         "file_format: \"0.1\"\n"
             + "tracer_provider:\n"
@@ -337,4 +344,149 @@ void nullValuesParsedToEmptyObjects() {
 
     assertThat(objectPlaceholderModel).isEqualTo(noObjectPlaceholderModel);
   }
+
+  @Test
+  void parse_nullBoxedPrimitivesParsedToNull() {
+    String yaml =
+        "file_format:\n" // String
+            + "disabled:\n" // Boolean
+            + "attribute_limits:\n"
+            + "  attribute_value_length_limit:\n" // Integer
+            + "tracer_provider:\n"
+            + "  sampler:\n"
+            + "    trace_id_ratio_based:\n"
+            + "      ratio:\n"; // Double
+
+    OpenTelemetryConfiguration model =
+        ConfigurationReader.parse(new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)));
+
+    assertThat(model.getFileFormat()).isNull();
+    assertThat(model.getDisabled()).isNull();
+    assertThat(model.getAttributeLimits().getAttributeValueLengthLimit()).isNull();
+    assertThat(model.getTracerProvider().getSampler().getTraceIdRatioBased().getRatio()).isNull();
+
+    assertThat(model)
+        .isEqualTo(
+            new OpenTelemetryConfiguration()
+                .withAttributeLimits(new AttributeLimits())
+                .withTracerProvider(
+                    new TracerProvider()
+                        .withSampler(
+                            new Sampler().withTraceIdRatioBased(new TraceIdRatioBased()))));
+  }
+
+  @ParameterizedTest
+  @MethodSource("envVarSubstitutionArgs")
+  void envSubstituteAndLoadYaml(
+      String rawYaml, String expectedSubstituteResult, Object expectedYamlResult) {
+    Map<String, String> environmentVariables = new HashMap<>();
+    environmentVariables.put("VAR_1", "value1");
+    environmentVariables.put("VAR_2", "value2");
+
+    String substitutedYaml =
+        ConfigurationReader.substituteEnvVariables(
+            new ByteArrayInputStream(rawYaml.getBytes(StandardCharsets.UTF_8)),
+            environmentVariables);
+    assertThat(substitutedYaml).isEqualTo(expectedSubstituteResult);
+
+    Object yaml =
+        ConfigurationReader.loadYaml(
+            new ByteArrayInputStream(rawYaml.getBytes(StandardCharsets.UTF_8)),
+            environmentVariables);
+    assertThat(yaml).isEqualTo(expectedYamlResult);
+  }
+
+  @SuppressWarnings("unchecked")
+  private static java.util.stream.Stream<Arguments> envVarSubstitutionArgs() {
+    return java.util.stream.Stream.of(
+        // Simple cases
+        Arguments.of("key1: ${env:VAR_1}", "key1: value1\n", mapOf(entry("key1", "value1"))),
+        Arguments.of(
+            "key1: ${env:VAR_1}\nkey2: value2\n",
+            "key1: value1\nkey2: value2\n",
+            mapOf(entry("key1", "value1"), entry("key2", "value2"))),
+        Arguments.of(
+            "key1: ${env:VAR_1} value1\nkey2: value2\n",
+            "key1: value1 value1\nkey2: value2\n",
+            mapOf(entry("key1", "value1 value1"), entry("key2", "value2"))),
+        // Multiple environment variables referenced
+        Arguments.of(
+            "key1: ${env:VAR_1}${env:VAR_2}\nkey2: value2\n",
+            "key1: value1value2\nkey2: value2\n",
+            mapOf(entry("key1", "value1value2"), entry("key2", "value2"))),
+        Arguments.of(
+            "key1: ${env:VAR_1} ${env:VAR_2}\nkey2: value2\n",
+            "key1: value1 value2\nkey2: value2\n",
+            mapOf(entry("key1", "value1 value2"), entry("key2", "value2"))),
+        // VAR_3 is not defined in environment
+        Arguments.of(
+            "key1: ${env:VAR_3}\nkey2: value2\n",
+            "key1: \nkey2: value2\n",
+            mapOf(entry("key1", null), entry("key2", "value2"))),
+        Arguments.of(
+            "key1: ${env:VAR_1} ${env:VAR_3}\nkey2: value2\n",
+            "key1: value1 \nkey2: value2\n",
+            mapOf(entry("key1", "value1"), entry("key2", "value2"))),
+        // Environment variable keys must match pattern: [a-zA-Z_]+[a-zA-Z0-9_]*
+        Arguments.of(
+            "key1: ${env:VAR&}\nkey2: value2\n",
+            "key1: ${env:VAR&}\nkey2: value2\n",
+            mapOf(entry("key1", "${env:VAR&}"), entry("key2", "value2"))));
+  }
+
+  private static <K, V> Map.Entry<K, V> entry(K key, @Nullable V value) {
+    return new AbstractMap.SimpleEntry<>(key, value);
+  }
+
+  @SuppressWarnings("unchecked")
+  private static Map<String, Object> mapOf(Map.Entry<String, ?>... entries) {
+    Map<String, Object> result = new HashMap<>();
+    for (Map.Entry<String, ?> entry : entries) {
+      result.put(entry.getKey(), entry.getValue());
+    }
+    return result;
+  }
+
+  @Test
+  void read_WithEnvironmentVariables() {
+    String yaml =
+        "file_format: \"0.1\"\n"
+            + "tracer_provider:\n"
+            + "  processors:\n"
+            + "    - batch:\n"
+            + "        exporter:\n"
+            + "          otlp:\n"
+            + "            endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT}\n"
+            + "    - batch:\n"
+            + "        exporter:\n"
+            + "          otlp:\n"
+            + "            endpoint: ${env:UNSET_ENV_VAR}\n";
+    Map<String, String> envVars = new HashMap<>();
+    envVars.put("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4317");
+    OpenTelemetryConfiguration model =
+        ConfigurationReader.parse(
+            new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)), envVars);
+    assertThat(model)
+        .isEqualTo(
+            new OpenTelemetryConfiguration()
+                .withFileFormat("0.1")
+                .withTracerProvider(
+                    new TracerProvider()
+                        .withProcessors(
+                            Arrays.asList(
+                                new SpanProcessor()
+                                    .withBatch(
+                                        new BatchSpanProcessor()
+                                            .withExporter(
+                                                new SpanExporter()
+                                                    .withOtlp(
+                                                        new Otlp()
+                                                            .withEndpoint(
+                                                                "http://collector:4317")))),
+                                new SpanProcessor()
+                                    .withBatch(
+                                        new BatchSpanProcessor()
+                                            .withExporter(
+                                                new SpanExporter().withOtlp(new Otlp())))))));
+  }
 }