diff --git a/application.example.yml b/application.example.yml index 9502e3fc5..0e2a03df6 100644 --- a/application.example.yml +++ b/application.example.yml @@ -295,4 +295,13 @@ akhq: - username: header-admin groups: - admin - + + # Data masking configuration + data-masking: + filters: + - description: "Masks value for secret-key fields" + search-regex: '"(secret-key)":".*"' + replacement: '"$1":"xxxx"' + - description: "Masks last digits of phone numbers" + search-regex: '"([\+]?[(]?[0-9]{3}[)]?[-\s\.]?[0-9]{3}[-\s\.]?)[0-9]{4,6}"' + replacement: '"$1xxxx"' diff --git a/docs/docs/configuration/akhq.md b/docs/docs/configuration/akhq.md index 33b6c78c0..c1baf26a6 100644 --- a/docs/docs/configuration/akhq.md +++ b/docs/docs/configuration/akhq.md @@ -67,3 +67,18 @@ akhq: value: "none" ``` +## Data Masking +If you want to hide some data in your records, you can configure this with the following filters. +These will be applied to all record values and keys. +```yaml +akhq: + security: + data-masking: + filters: + - description: "Masks value for secret-key fields" + search-regex: '"(secret-key)":".*"' + replacement: '"$1":"xxxx"' + - description: "Masks last digits of phone numbers" + search-regex: '"([\+]?[(]?[0-9]{3}[)]?[-\s\.]?[0-9]{3}[-\s\.]?)[0-9]{4,6}"' + replacement: '"$1xxxx"' +``` \ No newline at end of file diff --git a/src/main/java/org/akhq/configs/DataMasking.java b/src/main/java/org/akhq/configs/DataMasking.java new file mode 100644 index 000000000..6fb838e4f --- /dev/null +++ b/src/main/java/org/akhq/configs/DataMasking.java @@ -0,0 +1,13 @@ +package org.akhq.configs; + +import io.micronaut.context.annotation.ConfigurationProperties; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@ConfigurationProperties("akhq.security.data-masking") +@Data +public class DataMasking { + List filters = new ArrayList<>(); +} diff --git a/src/main/java/org/akhq/configs/DataMaskingFilter.java b/src/main/java/org/akhq/configs/DataMaskingFilter.java new file mode 100644 index 000000000..63075d541 --- /dev/null +++ b/src/main/java/org/akhq/configs/DataMaskingFilter.java @@ -0,0 +1,12 @@ +package org.akhq.configs; + +import io.micronaut.context.annotation.EachProperty; +import lombok.Data; + +@EachProperty("filters") +@Data +public class DataMaskingFilter { + String description; + String searchRegex; + String replacement; +} diff --git a/src/main/java/org/akhq/models/Record.java b/src/main/java/org/akhq/models/Record.java index 243eb1939..4e8739fc1 100644 --- a/src/main/java/org/akhq/models/Record.java +++ b/src/main/java/org/akhq/models/Record.java @@ -157,6 +157,10 @@ public void setValue(String value) { this.value = value; } + public void setKey(String key) { + this.key = key; + } + public void setTruncated(Boolean truncated) { this.truncated = truncated; } diff --git a/src/main/java/org/akhq/repositories/RecordRepository.java b/src/main/java/org/akhq/repositories/RecordRepository.java index ae073a750..20e8ccc1e 100644 --- a/src/main/java/org/akhq/repositories/RecordRepository.java +++ b/src/main/java/org/akhq/repositories/RecordRepository.java @@ -22,6 +22,7 @@ import org.akhq.modules.schemaregistry.RecordWithSchemaSerializerFactory; import org.akhq.utils.AvroToJsonSerializer; import org.akhq.utils.Debug; +import org.akhq.utils.MaskingUtils; import org.apache.kafka.clients.admin.DeletedRecords; import org.apache.kafka.clients.admin.RecordsToDelete; import org.apache.kafka.clients.consumer.*; @@ -72,6 +73,9 @@ public class RecordRepository extends AbstractRepository { @Inject private AvroWireFormatConverter avroWireFormatConverter; + @Inject + private MaskingUtils maskingUtils; + @Value("${akhq.topic-data.poll-timeout:1000}") protected int pollTimeout; @@ -443,7 +447,7 @@ private ConsumerRecords poll(KafkaConsumer consu private Record newRecord(ConsumerRecord record, String clusterId, Topic topic) { SchemaRegistryType schemaRegistryType = this.schemaRegistryRepository.getSchemaRegistryType(clusterId); SchemaRegistryClient client = this.kafkaModule.getRegistryClient(clusterId); - return new Record( + return maskingUtils.maskRecord(new Record( client, record, this.schemaRegistryRepository.getSchemaRegistryType(clusterId), @@ -456,13 +460,13 @@ private Record newRecord(ConsumerRecord record, String clusterId avroWireFormatConverter.convertValueToWireFormat(record, client, this.schemaRegistryRepository.getSchemaRegistryType(clusterId)), topic - ); + )); } private Record newRecord(ConsumerRecord record, BaseOptions options, Topic topic) { SchemaRegistryType schemaRegistryType = this.schemaRegistryRepository.getSchemaRegistryType(options.clusterId); SchemaRegistryClient client = this.kafkaModule.getRegistryClient(options.clusterId); - return new Record( + return maskingUtils.maskRecord(new Record( client, record, schemaRegistryType, @@ -475,7 +479,7 @@ private Record newRecord(ConsumerRecord record, BaseOptions opti avroWireFormatConverter.convertValueToWireFormat(record, client, this.schemaRegistryRepository.getSchemaRegistryType(options.clusterId)), topic - ); + )); } public List produce( diff --git a/src/main/java/org/akhq/utils/MaskingUtils.java b/src/main/java/org/akhq/utils/MaskingUtils.java new file mode 100644 index 000000000..71bae921d --- /dev/null +++ b/src/main/java/org/akhq/utils/MaskingUtils.java @@ -0,0 +1,38 @@ +package org.akhq.utils; + +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import org.akhq.configs.DataMasking; +import org.akhq.configs.DataMaskingFilter; +import org.akhq.models.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Singleton +public class MaskingUtils { + private static final Logger LOG = LoggerFactory.getLogger(MaskingUtils.class); + + @Inject + DataMasking dataMasking; + + public Record maskRecord(Record record) { + LOG.trace("masking record"); + + String value = record.getValue(); + String key = record.getKey(); + + for (DataMaskingFilter filter : dataMasking.getFilters()) { + if (value != null) { + value = value.replaceAll(filter.getSearchRegex(), filter.getReplacement()); + } + if (key != null) { + key = key.replaceAll(filter.getSearchRegex(), filter.getReplacement()); + } + } + + record.setValue(value); + record.setKey(key); + + return record; + } +} \ No newline at end of file diff --git a/src/test/java/org/akhq/utils/MaskingUtilsTest.java b/src/test/java/org/akhq/utils/MaskingUtilsTest.java new file mode 100644 index 000000000..26ff0758b --- /dev/null +++ b/src/test/java/org/akhq/utils/MaskingUtilsTest.java @@ -0,0 +1,64 @@ +package org.akhq.utils; + +import io.micronaut.context.ApplicationContext; +import org.akhq.models.Record; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class MaskingUtilsTest { + + @Test + void shouldMasksRecordValue() { + ApplicationContext ctx = ApplicationContext.run("data-masking"); + MaskingUtils maskingUtils = ctx.getBean(MaskingUtils.class); + + Record record = new Record(); + record.setValue("{\"secret-key\":\"my-secret-value\"}"); + + Record maskedRecord = maskingUtils.maskRecord(record); + + assertEquals( + "{\"secret-key\":\"xxxx\"}", + maskedRecord.getValue() + ); + + ctx.close(); + } + + @Test + void shouldMasksRecordKey() { + ApplicationContext ctx = ApplicationContext.run("data-masking"); + MaskingUtils maskingUtils = ctx.getBean(MaskingUtils.class); + + Record record = new Record(); + record.setKey("{\"secret-key\":\"my-secret-value\"}"); + + Record maskedRecord = maskingUtils.maskRecord(record); + + assertEquals( + "{\"secret-key\":\"xxxx\"}", + maskedRecord.getKey() + ); + + ctx.close(); + } + + @Test + void shouldReturnGroupsToAllowPartialMasking() { + ApplicationContext ctx = ApplicationContext.run("data-masking"); + MaskingUtils maskingUtils = ctx.getBean(MaskingUtils.class); + + Record record = new Record(); + record.setValue("{\"some-key\":\"+12092503766\"}"); + + Record maskedRecord = maskingUtils.maskRecord(record); + + assertEquals( + "{\"some-key\":\"+120925xxxx\"}", + maskedRecord.getValue() + ); + + ctx.close(); + } +} \ No newline at end of file diff --git a/src/test/resources/application-data-masking.yml b/src/test/resources/application-data-masking.yml new file mode 100644 index 000000000..e39d87e7a --- /dev/null +++ b/src/test/resources/application-data-masking.yml @@ -0,0 +1,10 @@ +akhq: + security: + data-masking: + filters: + - description: "Masks value for secret-key fields" + search-regex: '"(secret-key)":".*"' + replacement: '"$1":"xxxx"' + - description: "Masks last digits of phone numbers" + search-regex: '"([\+]?[(]?[0-9]{3}[)]?[-\s\.]?[0-9]{3}[-\s\.]?)[0-9]{4,6}"' + replacement: '"$1xxxx"' \ No newline at end of file