Skip to content

Commit

Permalink
Fix to support multiple property sources auto refresh (#41703)
Browse files Browse the repository at this point in the history
  • Loading branch information
@moarychan
moarychan authored Sep 6, 2024
1 parent 83dc72b commit 60f4093
Show file tree
Hide file tree
Showing 10 changed files with 476 additions and 362 deletions.
1 change: 1 addition & 0 deletions sdk/spring/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ This section includes changes in `spring-cloud-azure-autoconfigure` module.

#### Bugs Fixed
- Avoid always overriding the default binder when using Kafka binder. [#37337](https://github.com/Azure/azure-sdk-for-java/issues/37337).
- Fix to support multiple property sources auto refresh. [#26356](https://github.com/Azure/azure-sdk-for-java/issues/26356).

## 5.15.0 (2024-08-07)
- This release is compatible with Spring Boot 3.0.0-3.0.13, 3.1.0-3.1.12, 3.2.0-3.2.7, 3.3.0-3.3.2. (Note: 3.0.x (x>13), 3.1.y (y>12), 3.2.z (z>7) and 3.3.m (m>2) should be supported, but they aren't tested with this release.)
Expand Down
32 changes: 24 additions & 8 deletions ...d/autoconfigure/implementation/keyvault/environment/KeyVaultEnvironmentPostProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@
import org.springframework.util.StringUtils;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import static org.springframework.core.env.StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME;

Expand All @@ -47,8 +49,8 @@ public class KeyVaultEnvironmentPostProcessor implements EnvironmentPostProcesso
private static final String SKIP_CONFIGURE_REASON_FORMAT = "Skip configuring Key Vault PropertySource because %s.";

private final Log logger;
private final ConfigurableBootstrapContext bootstrapContext;

private final ConfigurableBootstrapContext bootstrapContext;

/**
* Creates a new instance of {@link KeyVaultEnvironmentPostProcessor}.
Expand Down Expand Up @@ -85,6 +87,9 @@ public void postProcessEnvironment(ConfigurableEnvironment environment, SpringAp
}

final List<AzureKeyVaultPropertySourceProperties> propertiesList = secretProperties.getPropertySources();

checkDuplicatePropertySourceNames(propertiesList);

List<KeyVaultPropertySource> keyVaultPropertySources = buildKeyVaultPropertySourceList(propertiesList);
final MutablePropertySources propertySources = environment.getPropertySources();
// reverse iterate order making sure smaller index has higher priority.
Expand All @@ -99,6 +104,16 @@ public void postProcessEnvironment(ConfigurableEnvironment environment, SpringAp
}
}

private void checkDuplicatePropertySourceNames(List<AzureKeyVaultPropertySourceProperties> propertiesList) {
List<String> sourceNames = propertiesList.stream()
.map(AzureKeyVaultPropertySourceProperties::getName)
.toList();
Set<String> deduplicatedSourceNames = new HashSet<>(sourceNames);
if (propertiesList.size() != deduplicatedSourceNames.size()) {
throw new IllegalStateException("Duplicate property source name found: " + sourceNames);
}
}

private List<KeyVaultPropertySource> buildKeyVaultPropertySourceList(
List<AzureKeyVaultPropertySourceProperties> propertiesList) {
List<KeyVaultPropertySource> propertySources = new ArrayList<>();
Expand All @@ -120,14 +135,15 @@ private List<KeyVaultPropertySource> buildKeyVaultPropertySourceList(
private KeyVaultPropertySource buildKeyVaultPropertySource(
AzureKeyVaultPropertySourceProperties properties) {
try {
final KeyVaultOperation keyVaultOperation = new KeyVaultOperation(
buildSecretClient(properties),
properties.getRefreshInterval(),
properties.getSecretKeys(),
properties.isCaseSensitive());
return new KeyVaultPropertySource(properties.getName(), keyVaultOperation);
final KeyVaultOperation keyVaultOperation = new KeyVaultOperation(buildSecretClient(properties));
return new KeyVaultPropertySource(
properties.getName(),
properties.getRefreshInterval(),
keyVaultOperation,
properties.getSecretKeys(),
properties.isCaseSensitive());
} catch (final Exception exception) {
throw new IllegalStateException("Failed to configure KeyVault property source", exception);
throw new IllegalStateException("Failed to configure KeyVault property source '" + properties.getName() + "'", exception);
}
}

Expand Down
185 changes: 22 additions & 163 deletions ...ure/spring/cloud/autoconfigure/implementation/keyvault/environment/KeyVaultOperation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,199 +8,58 @@
import com.azure.security.keyvault.secrets.SecretClient;
import com.azure.security.keyvault.secrets.models.KeyVaultSecret;
import com.azure.security.keyvault.secrets.models.SecretProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.lang.NonNull;

import java.time.Duration;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Timer;
import java.util.TimerTask;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

/**
* KeyVaultOperation wraps the operations to access Key Vault.
* This operation can list secrets with given filter, and convert it to spring property name and value.
*
* @since 4.0.0
*/
public class KeyVaultOperation {

private static final Logger LOGGER = LoggerFactory.getLogger(KeyVaultOperation.class);

/**
* Stores the case-sensitive flag.
*/
private final boolean caseSensitive;

/**
* Stores the properties.
*/
private Map<String, String> properties = new HashMap<>();

/**
* Stores the secret client.
*/
private final SecretClient secretClient;

/**
* Stores the secret keys.
*/
private final List<String> secretKeys;
/**
* Stores the timer object to schedule refresh task.
*/
private static Timer timer;

/**
* Constructor.
* @param secretClient the Key Vault secret client.
* @param refreshDuration the refresh in milliseconds (0 or less disables refresh).
* @param secretKeys the secret keys to look for.
* @param caseSensitive the case-sensitive flag.
*/
public KeyVaultOperation(final SecretClient secretClient,
final Duration refreshDuration,
List<String> secretKeys,
boolean caseSensitive) {

this.caseSensitive = caseSensitive;
public KeyVaultOperation(final SecretClient secretClient) {
this.secretClient = secretClient;
this.secretKeys = secretKeys;

refreshProperties();

final long refreshInMillis = refreshDuration.toMillis();
if (refreshInMillis > 0) {
synchronized (KeyVaultOperation.class) {
if (timer != null) {
try {
timer.cancel();
timer.purge();
} catch (RuntimeException runtimeException) {
LOGGER.error("Error of terminating Timer", runtimeException);
}
}
timer = new Timer(true);
final TimerTask task = new TimerTask() {
@Override
public void run() {
refreshProperties();
}
};
timer.scheduleAtFixedRate(task, refreshInMillis, refreshInMillis);
}
}
}

/**
* Get the property.
*
* @param property the property to get.
* @return the property value.
* Get the Key Vault secrets filtered by given secret keys.
* If the secret keys is empty, return all the secrets in Key Vault.
*/
public String getProperty(String property) {
return properties.get(toKeyVaultSecretName(property));
}

/**
* Get the property names.
*
* @return the property names.
*/
public String[] getPropertyNames() {
if (!caseSensitive) {
return properties
.keySet()
.stream()
.flatMap(p -> Stream.of(p, p.replace("-", ".")))
.distinct()
.toArray(String[]::new);
} else {
return properties
.keySet()
.toArray(new String[0]);
}
}

/**
* Refresh the properties by accessing key vault.
*/
private void refreshProperties() {
List<KeyVaultSecret> listSecrets(List<String> secretKeys) {
List<KeyVaultSecret> keyVaultSecrets;
if (secretKeys == null || secretKeys.isEmpty()) {
properties = Optional.of(secretClient)
.map(SecretClient::listPropertiesOfSecrets)
.map(ContinuablePagedIterable::iterableByPage)
.map(i -> StreamSupport.stream(i.spliterator(), false))
.orElseGet(Stream::empty)
.map(PagedResponse::getElements)
.flatMap(i -> StreamSupport.stream(i.spliterator(), false))
.filter(SecretProperties::isEnabled)
.map(p -> secretClient.getSecret(p.getName(), p.getVersion()))
.filter(Objects::nonNull)
.collect(Collectors.toMap(
s -> toKeyVaultSecretName(s.getName()),
KeyVaultSecret::getValue
));
} else {
properties = secretKeys.stream()
.map(this::toKeyVaultSecretName)
.map(secretClient::getSecret)
.filter(Objects::nonNull)
.collect(Collectors.toMap(
s -> toKeyVaultSecretName(s.getName()),
KeyVaultSecret::getValue
));
}
}

/**
* For convention, we need to support all relaxed binding format from spring, these may include:
* <table>
* <tr><td>Spring relaxed binding names</td></tr>
* <tr><td>acme.my-project.person.first-name</td></tr>
* <tr><td>acme.myProject.person.firstName</td></tr>
* <tr><td>acme.my_project.person.first_name</td></tr>
* <tr><td>ACME_MYPROJECT_PERSON_FIRSTNAME</td></tr>
* </table>
* But azure key vault only allows ^[0-9a-zA-Z-]+$ and case-insensitive, so
* there must be some conversion between spring names and azure key vault
* names. For example, the 4 properties stated above should be converted to
* acme-myproject-person-firstname in key vault.
*
* @param property of secret instance.
* @return the value of secret with given name or null.
*/
private String toKeyVaultSecretName(@NonNull String property) {
if (!caseSensitive) {
if (property.matches("[a-z0-9A-Z-]+")) {
return property.toLowerCase(Locale.US);
} else if (property.matches("[A-Z0-9_]+")) {
return property.toLowerCase(Locale.US).replace("_", "-");
} else {
return property.toLowerCase(Locale.US)
.replace("-", "") // my-project -> myproject
.replace("_", "") // my_project -> myproject
.replace(".", "-"); // acme.myproject -> acme-myproject
}
keyVaultSecrets = Optional.of(secretClient)
.map(SecretClient::listPropertiesOfSecrets)
.map(ContinuablePagedIterable::iterableByPage)
.map(i -> StreamSupport.stream(i.spliterator(), false))
.orElseGet(Stream::empty)
.map(PagedResponse::getElements)
.flatMap(i -> StreamSupport.stream(i.spliterator(), false))
.filter(SecretProperties::isEnabled)
.map(p -> secretClient.getSecret(p.getName(), p.getVersion()))
.filter(Objects::nonNull)
.toList();
} else {
return property;
keyVaultSecrets = secretKeys.stream()
.map(secretClient::getSecret)
.filter(Objects::nonNull)
.toList();
}
return keyVaultSecrets;
}

/**
* Set the properties.
*
* @param properties the properties.
*/
void setProperties(HashMap<String, String> properties) {
this.properties = properties;
}

}
Loading

0 comments on commit 60f4093

Please sign in to comment.