Skip to content

Commit

Permalink
Add env var substitution support to file configuration (#5914)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jack-berg
jack-berg authored Oct 23, 2023
1 parent 5740c3c commit e592d5d
Show file tree
Hide file tree
Showing 3 changed files with 252 additions and 10 deletions.
3 changes: 3 additions & 0 deletions ...c/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ private ConfigurationFactory() {}
* Parse the {@code inputStream} YAML to {@link OpenTelemetryConfiguration} and interpret the
* model to create {@link OpenTelemetrySdk} instance corresponding to the configuration.
*
* <p>Before parsing, environment variable substitution is performed as described in {@link
* ConfigurationReader.EnvSubstitutionConstructor}.
*
* @param inputStream the configuration YAML
* @return the {@link OpenTelemetrySdk}
*/
Expand Down
113 changes: 105 additions & 8 deletions ...rc/main/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,23 +10,120 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import io.opentelemetry.sdk.extension.incubator.fileconfig.internal.model.OpenTelemetryConfiguration;
import java.io.InputStream;
import java.util.Map;
import java.util.regex.MatchResult;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.snakeyaml.engine.v2.api.Load;
import org.snakeyaml.engine.v2.api.LoadSettings;
import org.snakeyaml.engine.v2.constructor.StandardConstructor;
import org.snakeyaml.engine.v2.nodes.MappingNode;
import org.yaml.snakeyaml.Yaml;

final class ConfigurationReader {

private static final ObjectMapper MAPPER =
new ObjectMapper()
// Create empty object instances for keys which are present but have null values
.setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY));
private static final Pattern ENV_VARIABLE_REFERENCE =
Pattern.compile("\\$\\{env:([a-zA-Z_]+[a-zA-Z0-9_]*)}");

private static final ObjectMapper MAPPER;

static {
MAPPER =
new ObjectMapper()
// Create empty object instances for keys which are present but have null values
.setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY));
// Boxed primitives which are present but have null values should be set to null, rather than
// empty instances
MAPPER.configOverride(String.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
MAPPER.configOverride(Integer.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
MAPPER.configOverride(Double.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
MAPPER.configOverride(Boolean.class).setSetterInfo(JsonSetter.Value.forValueNulls(Nulls.SET));
}

private ConfigurationReader() {}

/** Parse the {@code configuration} YAML and return the {@link OpenTelemetryConfiguration}. */
/**
* Parse the {@code configuration} YAML and return the {@link OpenTelemetryConfiguration}.
*
* <p>Before parsing, environment variable substitution is performed as described in {@link
* EnvSubstitutionConstructor}.
*/
static OpenTelemetryConfiguration parse(InputStream configuration) {
LoadSettings settings = LoadSettings.builder().build();
Load yaml = new Load(settings);
Object yamlObj = yaml.loadFromInputStream(configuration);
return parse(configuration, System.getenv());
}

// Visible for testing
static OpenTelemetryConfiguration parse(
InputStream configuration, Map<String, String> environmentVariables) {
Object yamlObj = loadYaml(configuration, environmentVariables);
return MAPPER.convertValue(yamlObj, OpenTelemetryConfiguration.class);
}

static Object loadYaml(InputStream inputStream, Map<String, String> environmentVariables) {
LoadSettings settings = LoadSettings.builder().build();
Load yaml = new Load(settings, new EnvSubstitutionConstructor(settings, environmentVariables));
return yaml.loadFromInputStream(inputStream);
}

/**
* {@link StandardConstructor} which substitutes environment variables.
*
* <p>Environment variables follow the syntax {@code ${env:VARIABLE}}, where {@code VARIABLE} is
* an environment variable matching the regular expression {@code [a-zA-Z_]+[a-zA-Z0-9_]*}.
*
* <p>Environment variable substitution only takes place on scalar values of maps. References to
* environment variables in keys or sets are ignored.
*
* <p>If a referenced environment variable is not defined, it is replaced with {@code ""}.
*/
static final class EnvSubstitutionConstructor extends StandardConstructor {

// Yaml is not thread safe but this instance is always used on the same thread
private final Yaml yaml = new Yaml();
private final Map<String, String> environmentVariables;

private EnvSubstitutionConstructor(
LoadSettings loadSettings, Map<String, String> environmentVariables) {
super(loadSettings);
this.environmentVariables = environmentVariables;
}

@Override
protected Map<Object, Object> constructMapping(MappingNode node) {
// First call the super to construct mapping from MappingNode as usual
Map<Object, Object> result = super.constructMapping(node);

// Iterate through the map entries, and:
// 1. Identify entries which are scalar strings eligible for environment variable substitution
// 2. Apply environment variable substitution
// 3. Re-parse substituted value so it has correct type (i.e. yaml.load(newVal))
for (Map.Entry<Object, Object> entry : result.entrySet()) {
Object value = entry.getValue();
if (!(value instanceof String)) {
continue;
}

String val = (String) value;
Matcher matcher = ENV_VARIABLE_REFERENCE.matcher(val);
if (!matcher.find()) {
continue;
}

int offset = 0;
StringBuilder newVal = new StringBuilder();
do {
MatchResult matchResult = matcher.toMatchResult();
String replacement = environmentVariables.getOrDefault(matcher.group(1), "");
newVal.append(val, offset, matchResult.start()).append(replacement);
offset = matchResult.end();
} while (matcher.find());
if (offset != val.length()) {
newVal.append(val, offset, val.length());
}
entry.setValue(yaml.load(newVal.toString()));
}

return result;
}
}
}
146 changes: 144 additions & 2 deletions ...est/java/io/opentelemetry/sdk/extension/incubator/fileconfig/ConfigurationReaderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,21 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.AbstractMap;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.Nullable;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;

class ConfigurationReaderTest {

@Test
void read_KitchenSinkExampleFile() throws IOException {
void parse_KitchenSinkExampleFile() throws IOException {
OpenTelemetryConfiguration expected = new OpenTelemetryConfiguration();

expected.withFileFormat("0.1");
Expand Down Expand Up @@ -283,7 +290,7 @@ void read_KitchenSinkExampleFile() throws IOException {
}

@Test
void nullValuesParsedToEmptyObjects() {
void parse_nullValuesParsedToEmptyObjects() {
String objectPlaceholderString =
"file_format: \"0.1\"\n"
+ "tracer_provider:\n"
Expand Down Expand Up @@ -337,4 +344,139 @@ void nullValuesParsedToEmptyObjects() {

assertThat(objectPlaceholderModel).isEqualTo(noObjectPlaceholderModel);
}

@Test
void parse_nullBoxedPrimitivesParsedToNull() {
String yaml =
"file_format:\n" // String
+ "disabled:\n" // Boolean
+ "attribute_limits:\n"
+ " attribute_value_length_limit:\n" // Integer
+ "tracer_provider:\n"
+ " sampler:\n"
+ " trace_id_ratio_based:\n"
+ " ratio:\n"; // Double

OpenTelemetryConfiguration model =
ConfigurationReader.parse(new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)));

assertThat(model.getFileFormat()).isNull();
assertThat(model.getDisabled()).isNull();
assertThat(model.getAttributeLimits().getAttributeValueLengthLimit()).isNull();
assertThat(model.getTracerProvider().getSampler().getTraceIdRatioBased().getRatio()).isNull();

assertThat(model)
.isEqualTo(
new OpenTelemetryConfiguration()
.withAttributeLimits(new AttributeLimits())
.withTracerProvider(
new TracerProvider()
.withSampler(
new Sampler().withTraceIdRatioBased(new TraceIdRatioBased()))));
}

@ParameterizedTest
@MethodSource("envVarSubstitutionArgs")
void envSubstituteAndLoadYaml(String rawYaml, Object expectedYamlResult) {
Map<String, String> environmentVariables = new HashMap<>();
environmentVariables.put("STR_1", "value1");
environmentVariables.put("STR_2", "value2");
environmentVariables.put("BOOL", "true");
environmentVariables.put("INT", "1");
environmentVariables.put("FLOAT", "1.1");

Object yaml =
ConfigurationReader.loadYaml(
new ByteArrayInputStream(rawYaml.getBytes(StandardCharsets.UTF_8)),
environmentVariables);
assertThat(yaml).isEqualTo(expectedYamlResult);
}

@SuppressWarnings("unchecked")
private static java.util.stream.Stream<Arguments> envVarSubstitutionArgs() {
return java.util.stream.Stream.of(
// Simple cases
Arguments.of("key1: ${env:STR_1}\n", mapOf(entry("key1", "value1"))),
Arguments.of("key1: ${env:BOOL}\n", mapOf(entry("key1", true))),
Arguments.of("key1: ${env:INT}\n", mapOf(entry("key1", 1))),
Arguments.of("key1: ${env:FLOAT}\n", mapOf(entry("key1", 1.1))),
Arguments.of(
"key1: ${env:STR_1}\n" + "key2: value2\n",
mapOf(entry("key1", "value1"), entry("key2", "value2"))),
Arguments.of(
"key1: ${env:STR_1} value1\n" + "key2: value2\n",
mapOf(entry("key1", "value1 value1"), entry("key2", "value2"))),
// Multiple environment variables referenced
Arguments.of("key1: ${env:STR_1}${env:STR_2}\n", mapOf(entry("key1", "value1value2"))),
Arguments.of("key1: ${env:STR_1} ${env:STR_2}\n", mapOf(entry("key1", "value1 value2"))),
// Undefined environment variable
Arguments.of("key1: ${env:STR_3}\n", mapOf(entry("key1", null))),
Arguments.of("key1: ${env:STR_1} ${env:STR_3}\n", mapOf(entry("key1", "value1"))),
// Environment variable keys must match pattern: [a-zA-Z_]+[a-zA-Z0-9_]*
Arguments.of("key1: ${env:VAR&}\n", mapOf(entry("key1", "${env:VAR&}"))),
// Environment variable substitution only takes place in scalar values of maps
Arguments.of("${env:STR_1}: value1\n", mapOf(entry("${env:STR_1}", "value1"))),
Arguments.of(
"key1:\n ${env:STR_1}: value1\n",
mapOf(entry("key1", mapOf(entry("${env:STR_1}", "value1"))))),
Arguments.of(
"key1:\n - ${env:STR_1}\n",
mapOf(entry("key1", Collections.singletonList("${env:STR_1}")))));
}

private static <K, V> Map.Entry<K, V> entry(K key, @Nullable V value) {
return new AbstractMap.SimpleEntry<>(key, value);
}

@SuppressWarnings("unchecked")
private static Map<String, Object> mapOf(Map.Entry<String, ?>... entries) {
Map<String, Object> result = new HashMap<>();
for (Map.Entry<String, ?> entry : entries) {
result.put(entry.getKey(), entry.getValue());
}
return result;
}

@Test
void read_WithEnvironmentVariables() {
String yaml =
"file_format: \"0.1\"\n"
+ "tracer_provider:\n"
+ " processors:\n"
+ " - batch:\n"
+ " exporter:\n"
+ " otlp:\n"
+ " endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT}\n"
+ " - batch:\n"
+ " exporter:\n"
+ " otlp:\n"
+ " endpoint: \"${env:UNSET_ENV_VAR}\"\n";
Map<String, String> envVars = new HashMap<>();
envVars.put("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4317");
OpenTelemetryConfiguration model =
ConfigurationReader.parse(
new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)), envVars);
assertThat(model)
.isEqualTo(
new OpenTelemetryConfiguration()
.withFileFormat("0.1")
.withTracerProvider(
new TracerProvider()
.withProcessors(
Arrays.asList(
new SpanProcessor()
.withBatch(
new BatchSpanProcessor()
.withExporter(
new SpanExporter()
.withOtlp(
new Otlp()
.withEndpoint(
"http://collector:4317")))),
new SpanProcessor()
.withBatch(
new BatchSpanProcessor()
.withExporter(
new SpanExporter().withOtlp(new Otlp())))))));
}
}

0 comments on commit e592d5d

Please sign in to comment.