From 7e8ef4118f39b3b28eb00245b06da6036dd4d044 Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Sat, 6 Jan 2024 00:06:20 -0600 Subject: [PATCH 01/12] configuration and serializers/desieralizers --- .../GlueSchemaRegistryConfiguration.java | 16 +++++++++++++ .../utils/AWSSchemaRegistryConstants.java | 7 ++++++ .../GlueSchemaRegistryConfigurationTest.java | 23 +++++++++++++++---- .../deserializers/json/JsonDeserializer.java | 7 ++++++ .../serializers/json/JsonSerializer.java | 5 ++++ .../json/JsonDeserializerTest.java | 8 ++++--- 6 files changed, 58 insertions(+), 8 deletions(-) diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java index ce7d870d..b0d367df 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java @@ -22,6 +22,7 @@ import com.amazonaws.services.schemaregistry.utils.ProtobufMessageType; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.module.SimpleModule; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.EnumUtils; @@ -58,6 +59,7 @@ public class GlueSchemaRegistryConfiguration { private Map metadata; private String secondaryDeserializer; private URI proxyUrl; + private SimpleModule javaTimeModule; /** * Name of the application using the serializer/deserializer. @@ -104,6 +106,7 @@ private void buildSchemaRegistryConfigs(Map configs) { validateAndSetUserAgent(configs); validateAndSetSecondaryDeserializer(configs); validateAndSetProxyUrl(configs); + validateAndSetJavaTimeModule(configs); } private void validateAndSetSecondaryDeserializer(Map configs) { @@ -323,6 +326,19 @@ private void validateAndSetJacksonDeserializationFeatures(Map configs } } + private void validateAndSetJavaTimeModule(Map configs) { + if (isPresent(configs, AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE)) { + String moduleClassName = String.valueOf(configs.get(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE)); + try { + Class moduleClass = Class.forName(moduleClassName); + this.javaTimeModule = (SimpleModule) moduleClass.getConstructor().newInstance(); + } catch (Exception e) { + String message = String.format("Invalid JavaTimeModule specified: %s", moduleClassName); + throw new AWSSchemaRegistryException(message, e); + } + } + } + private boolean isPresent(Map configs, String key) { if (!GlueSchemaRegistryUtils.getInstance() diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/utils/AWSSchemaRegistryConstants.java b/common/src/main/java/com/amazonaws/services/schemaregistry/utils/AWSSchemaRegistryConstants.java index 5ce30b06..a5867709 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/utils/AWSSchemaRegistryConstants.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/utils/AWSSchemaRegistryConstants.java @@ -172,6 +172,13 @@ public final class AWSSchemaRegistryConstants { */ public static final String USER_AGENT_APP = "userAgentApp"; + /** + * Jackson Java Time Module to use for handling Java 8 Date/Time. + * By default, no module is registered. + * Ex: com.fasterxml.jackson.datatype.jsr310.JavaTimeModule + */ + public static final String REGISTER_JAVA_TIME_MODULE = "registerJavaTimeModule"; + /** * Private constructor to avoid initialization of the class. */ diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java index 249c4605..de5efc57 100644 --- a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java @@ -32,11 +32,7 @@ import java.util.Map; import java.util.Properties; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; /** * Unit tests for testing configuration elements. @@ -410,4 +406,21 @@ public void testBuildConfig_invalidProxyUrl_throwsException() { Exception exception = assertThrows(AWSSchemaRegistryException.class, () -> new GlueSchemaRegistryConfiguration(props)); assertEquals("Proxy URL property is not a valid URL: "+proxy, exception.getMessage()); } + + @Test + public void testBuildConfig_defaultJavaTimeModule_succeeds() { + Properties props = createTestProperties(); + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); + assertNull(glueSchemaRegistryConfiguration.getJavaTimeModule()); + } + + @Test + public void testBuildConfig_javaTimeModuleEnabledWithoutDependency_throwException() { + Properties props = createTestProperties(); + String moduleClassName = "com.fasterxml.jackson.datatype.jsr310.JavaTimeModule"; + props.put(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE, moduleClassName); + Exception exception = assertThrows(AWSSchemaRegistryException.class, () -> new GlueSchemaRegistryConfiguration(props)); + String message = String.format("Invalid JavaTimeModule specified: %s", moduleClassName); + assertEquals(message, exception.getMessage()); + } } \ No newline at end of file diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java index 5c0c60a5..cf0fc606 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java @@ -22,7 +22,9 @@ import com.amazonaws.services.schemaregistry.common.Schema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.module.SimpleModule; import lombok.Builder; import lombok.Data; import lombok.Getter; @@ -32,6 +34,7 @@ import org.apache.commons.collections4.CollectionUtils; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; /** @@ -68,6 +71,10 @@ public JsonDeserializer(GlueSchemaRegistryConfiguration configs) { configs.getJacksonDeserializationFeatures() .forEach(this.objectMapper::enable); } + if (configs.getJavaTimeModule() != null) { + this.objectMapper.registerModule(configs.getJavaTimeModule()); + this.objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + } } } diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java index 063bbf11..db4db603 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.kjetland.jackson.jsonSchema.JsonSchemaGenerator; import lombok.Builder; @@ -63,6 +64,10 @@ public JsonSerializer(GlueSchemaRegistryConfiguration configs) { configs.getJacksonDeserializationFeatures() .forEach(this.objectMapper::enable); } + if (configs.getJavaTimeModule() != null) { + this.objectMapper.registerModule(configs.getJavaTimeModule()); + this.objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + } } this.jsonSchemaGenerator = new JsonSchemaGenerator(this.objectMapper); } diff --git a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializerTest.java b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializerTest.java index 6a7d63b4..14168ff0 100644 --- a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializerTest.java +++ b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializerTest.java @@ -26,7 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; public class JsonDeserializerTest { - private JsonDeserializer jsonDeserializer = new JsonDeserializer(null); + private JsonDeserializer defaultJsonDeserializer = new JsonDeserializer(null); @Test public void testDeserialize_nullArgs_throwsException() { @@ -42,8 +42,10 @@ public void testDeserialize_nullArgs_throwsException() { Schema testSchema = new Schema(testSchemaDefinition, DataFormat.JSON.name(), "testJson"); - assertThrows(IllegalArgumentException.class, () -> jsonDeserializer.deserialize(null, testSchema)); - assertThrows(IllegalArgumentException.class, () -> jsonDeserializer.deserialize(ByteBuffer.wrap(testBytes), + assertThrows(IllegalArgumentException.class, () -> defaultJsonDeserializer.deserialize(null, testSchema)); + assertThrows(IllegalArgumentException.class, () -> defaultJsonDeserializer.deserialize(ByteBuffer.wrap(testBytes), null)); } + + } From f9634f113c29c3c7f0d8d5030e38d74df3e5b097 Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Sat, 6 Jan 2024 09:17:01 -0600 Subject: [PATCH 02/12] create javatimemodule each time rather than share across objectmapper instances --- .../GlueSchemaRegistryConfiguration.java | 21 +++++++----- .../deserializers/json/JsonDeserializer.java | 18 ++--------- .../serializers/json/JsonSerializer.java | 18 ++--------- .../utils/json/ObjectMapperUtils.java | 32 +++++++++++++++++++ 4 files changed, 49 insertions(+), 40 deletions(-) create mode 100644 serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java index b0d367df..5ed5a56a 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java @@ -59,7 +59,7 @@ public class GlueSchemaRegistryConfiguration { private Map metadata; private String secondaryDeserializer; private URI proxyUrl; - private SimpleModule javaTimeModule; + private String javaTimeModuleClass; /** * Name of the application using the serializer/deserializer. @@ -329,13 +329,18 @@ private void validateAndSetJacksonDeserializationFeatures(Map configs private void validateAndSetJavaTimeModule(Map configs) { if (isPresent(configs, AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE)) { String moduleClassName = String.valueOf(configs.get(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE)); - try { - Class moduleClass = Class.forName(moduleClassName); - this.javaTimeModule = (SimpleModule) moduleClass.getConstructor().newInstance(); - } catch (Exception e) { - String message = String.format("Invalid JavaTimeModule specified: %s", moduleClassName); - throw new AWSSchemaRegistryException(message, e); - } + this.javaTimeModuleClass = moduleClassName; + loadJavaTimeModule(); + } + } + + public SimpleModule loadJavaTimeModule() { + try { + Class moduleClass = Class.forName(this.javaTimeModuleClass); + return (SimpleModule) moduleClass.getConstructor().newInstance(); + } catch (Exception e) { + String message = String.format("Invalid JavaTimeModule specified: %s", this.javaTimeModuleClass); + throw new AWSSchemaRegistryException(message, e); } } diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java index cf0fc606..27e2b694 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java @@ -20,6 +20,7 @@ import com.amazonaws.services.schemaregistry.exception.AWSSchemaRegistryException; import com.amazonaws.services.schemaregistry.serializers.json.JsonDataWithSchema; import com.amazonaws.services.schemaregistry.common.Schema; +import com.amazonaws.services.schemaregistry.utils.json.ObjectMapperUtils; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; @@ -60,22 +61,7 @@ public class JsonDeserializer implements GlueSchemaRegistryDataFormatDeserialize public JsonDeserializer(GlueSchemaRegistryConfiguration configs) { this.schemaRegistrySerDeConfigs = configs; JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(true); - this.objectMapper = new ObjectMapper(); - this.objectMapper.setNodeFactory(jsonNodeFactory); - if (configs != null) { - if (!CollectionUtils.isEmpty(configs.getJacksonSerializationFeatures())) { - configs.getJacksonSerializationFeatures() - .forEach(this.objectMapper::enable); - } - if (!CollectionUtils.isEmpty(configs.getJacksonDeserializationFeatures())) { - configs.getJacksonDeserializationFeatures() - .forEach(this.objectMapper::enable); - } - if (configs.getJavaTimeModule() != null) { - this.objectMapper.registerModule(configs.getJavaTimeModule()); - this.objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - } - } + this.objectMapper = ObjectMapperUtils.create(configs, jsonNodeFactory); } /** diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java index db4db603..0777ef88 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java @@ -17,6 +17,7 @@ import com.amazonaws.services.schemaregistry.common.GlueSchemaRegistryDataFormatSerializer; import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; import com.amazonaws.services.schemaregistry.exception.AWSSchemaRegistryException; +import com.amazonaws.services.schemaregistry.utils.json.ObjectMapperUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -53,22 +54,7 @@ public class JsonSerializer implements GlueSchemaRegistryDataFormatSerializer { public JsonSerializer(GlueSchemaRegistryConfiguration configs) { this.schemaRegistrySerDeConfigs = configs; JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(true); - this.objectMapper = new ObjectMapper(); - this.objectMapper.setNodeFactory(jsonNodeFactory); - if (configs != null) { - if (!CollectionUtils.isEmpty(configs.getJacksonSerializationFeatures())) { - configs.getJacksonSerializationFeatures() - .forEach(this.objectMapper::enable); - } - if (!CollectionUtils.isEmpty(configs.getJacksonDeserializationFeatures())) { - configs.getJacksonDeserializationFeatures() - .forEach(this.objectMapper::enable); - } - if (configs.getJavaTimeModule() != null) { - this.objectMapper.registerModule(configs.getJavaTimeModule()); - this.objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - } - } + this.objectMapper = ObjectMapperUtils.create(configs, jsonNodeFactory); this.jsonSchemaGenerator = new JsonSchemaGenerator(this.objectMapper); } diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java new file mode 100644 index 00000000..78b31f0d --- /dev/null +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java @@ -0,0 +1,32 @@ +package com.amazonaws.services.schemaregistry.utils.json; + + +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import org.apache.commons.collections4.CollectionUtils; + +public class ObjectMapperUtils { + + public static ObjectMapper create(GlueSchemaRegistryConfiguration configs, JsonNodeFactory jsonNodeFactory) { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setNodeFactory(jsonNodeFactory); + if (configs != null) { + if (!CollectionUtils.isEmpty(configs.getJacksonSerializationFeatures())) { + configs.getJacksonSerializationFeatures() + .forEach(objectMapper::enable); + } + if (!CollectionUtils.isEmpty(configs.getJacksonDeserializationFeatures())) { + configs.getJacksonDeserializationFeatures() + .forEach(objectMapper::enable); + } + if (configs.getJavaTimeModuleClass() != null) { + objectMapper.registerModule(configs.loadJavaTimeModule()); + objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + } + } + + return objectMapper; + } +} From a61c4ca402a3a98e5e70a250569d0a49217e5575 Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Sun, 7 Jan 2024 09:05:26 -0600 Subject: [PATCH 03/12] added and tested java8 datetime support --- .../GlueSchemaRegistryConfigurationTest.java | 2 +- integration-tests/pom.xml | 4 ++ .../ObjectMapperDateTimeSupportTest.java | 71 +++++++++++++++++++ ...eSchemaRegistryKinesisIntegrationTest.java | 2 +- .../deserializers/json/JsonDeserializer.java | 8 +-- .../serializers/json/JsonSerializer.java | 6 +- .../utils/json/ObjectMapperUtils.java | 6 +- .../utils/json/ObjectMapperUtilsTest.java | 30 ++++++++ 8 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/kafka/ObjectMapperDateTimeSupportTest.java create mode 100644 serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtilsTest.java diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java index de5efc57..fca11387 100644 --- a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java @@ -411,7 +411,7 @@ public void testBuildConfig_invalidProxyUrl_throwsException() { public void testBuildConfig_defaultJavaTimeModule_succeeds() { Properties props = createTestProperties(); GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); - assertNull(glueSchemaRegistryConfiguration.getJavaTimeModule()); + assertNull(glueSchemaRegistryConfiguration.loadJavaTimeModule()); } @Test diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index c7e76867..87849321 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -176,6 +176,10 @@ org.hamcrest hamcrest-all + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + javax.xml.bind jaxb-api diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/kafka/ObjectMapperDateTimeSupportTest.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/kafka/ObjectMapperDateTimeSupportTest.java new file mode 100644 index 00000000..bf37405f --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/kafka/ObjectMapperDateTimeSupportTest.java @@ -0,0 +1,71 @@ +package com.amazonaws.services.schemaregistry.integrationtests.kafka; + +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import com.amazonaws.services.schemaregistry.utils.json.ObjectMapperUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.InvalidDefinitionException; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class ObjectMapperDateTimeSupportTest { + + @Test + void testCreate_useJavaTimeModule_succeeds() { + Map map = new HashMap<>(); + map.put(AWSSchemaRegistryConstants.AWS_REGION, "US-West-1"); + map.put(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE, "com.fasterxml.jackson.datatype.jsr310.JavaTimeModule"); + GlueSchemaRegistryConfiguration cfg = new GlueSchemaRegistryConfiguration(map); + + ObjectMapper om = ObjectMapperUtils.create(cfg); + assertNotNull(cfg.loadJavaTimeModule()); + assertNotNull(om); + } + + @Test + void testCreate_noJavaTimeModule_failsSerializingJava8DateTime() throws JsonProcessingException { + Map map = new HashMap<>(); + map.put(AWSSchemaRegistryConstants.AWS_REGION, "US-West-1"); + GlueSchemaRegistryConfiguration cfg = new GlueSchemaRegistryConfiguration(map); + + ObjectMapper om = ObjectMapperUtils.create(cfg); + + MyTestObject myObj = new MyTestObject("obj1", LocalDateTime.now()); + + assertThrows(InvalidDefinitionException.class, () -> om.writeValueAsString(myObj)); + } + + @Test + void testCreate_withJavaTimeModule_succeedsSerializingAndDeserializingJava8DateTime() throws JsonProcessingException { + Map map = new HashMap<>(); + map.put(AWSSchemaRegistryConstants.AWS_REGION, "US-West-1"); + map.put(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE, "com.fasterxml.jackson.datatype.jsr310.JavaTimeModule"); + GlueSchemaRegistryConfiguration cfg = new GlueSchemaRegistryConfiguration(map); + + ObjectMapper om = ObjectMapperUtils.create(cfg); + + MyTestObject expectedObj = new MyTestObject("obj1", LocalDateTime.now()); + String jsonStr = om.writeValueAsString(expectedObj); + + MyTestObject actualObj = om.readValue(jsonStr, MyTestObject.class); + assertNotNull(actualObj); + assertEquals(expectedObj.getCreated(), actualObj.getCreated()); + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + static class MyTestObject { + String name; + LocalDateTime created; + } +} diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/kinesis/GlueSchemaRegistryKinesisIntegrationTest.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/kinesis/GlueSchemaRegistryKinesisIntegrationTest.java index 68a03098..be77c330 100644 --- a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/kinesis/GlueSchemaRegistryKinesisIntegrationTest.java +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/kinesis/GlueSchemaRegistryKinesisIntegrationTest.java @@ -496,7 +496,7 @@ private String produceRecordsWithKPL(String streamName, byte[] serializedBytes = dataFormatSerializer.serialize(record); putFutures.add(producer.addUserRecord(streamName, Long.toString(timestamp.toEpochMilli()), null, - ByteBuffer.wrap(serializedBytes), gsrSchema)); + ByteBuffer.wrap(serializedBytes), null, gsrSchema)); } String shardId = null; diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java index 27e2b694..3a8231e4 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializer.java @@ -23,19 +23,14 @@ import com.amazonaws.services.schemaregistry.utils.json.ObjectMapperUtils; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.module.SimpleModule; import lombok.Builder; import lombok.Data; import lombok.Getter; import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; /** @@ -60,8 +55,7 @@ public class JsonDeserializer implements GlueSchemaRegistryDataFormatDeserialize @Builder public JsonDeserializer(GlueSchemaRegistryConfiguration configs) { this.schemaRegistrySerDeConfigs = configs; - JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(true); - this.objectMapper = ObjectMapperUtils.create(configs, jsonNodeFactory); + this.objectMapper = ObjectMapperUtils.create(configs); } /** diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java index 0777ef88..c945e51a 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java @@ -21,15 +21,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.kjetland.jackson.jsonSchema.JsonSchemaGenerator; import lombok.Builder; import lombok.Getter; import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import java.nio.charset.StandardCharsets; @@ -53,8 +50,7 @@ public class JsonSerializer implements GlueSchemaRegistryDataFormatSerializer { @Builder public JsonSerializer(GlueSchemaRegistryConfiguration configs) { this.schemaRegistrySerDeConfigs = configs; - JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(true); - this.objectMapper = ObjectMapperUtils.create(configs, jsonNodeFactory); + this.objectMapper = ObjectMapperUtils.create(configs); this.jsonSchemaGenerator = new JsonSchemaGenerator(this.objectMapper); } diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java index 78b31f0d..85f79f0c 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java @@ -1,6 +1,5 @@ package com.amazonaws.services.schemaregistry.utils.json; - import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; @@ -9,9 +8,10 @@ public class ObjectMapperUtils { - public static ObjectMapper create(GlueSchemaRegistryConfiguration configs, JsonNodeFactory jsonNodeFactory) { + public static ObjectMapper create(GlueSchemaRegistryConfiguration configs) { ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.setNodeFactory(jsonNodeFactory); + objectMapper.setNodeFactory(JsonNodeFactory.withExactBigDecimals(true)); + if (configs != null) { if (!CollectionUtils.isEmpty(configs.getJacksonSerializationFeatures())) { configs.getJacksonSerializationFeatures() diff --git a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtilsTest.java b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtilsTest.java new file mode 100644 index 00000000..14f95b20 --- /dev/null +++ b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtilsTest.java @@ -0,0 +1,30 @@ +package com.amazonaws.services.schemaregistry.utils.json; + +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Unit tests for testing object mapper + */ +public class ObjectMapperUtilsTest { + + static GlueSchemaRegistryConfiguration testConfigs(Map configs) { + configs.putIfAbsent(AWSSchemaRegistryConstants.AWS_REGION, "US-West-1"); + return new GlueSchemaRegistryConfiguration(configs); + } + + @Test + void testCreate_succeeds() { + GlueSchemaRegistryConfiguration cfg = testConfigs(new HashMap()); + + ObjectMapper om = ObjectMapperUtils.create(cfg); + assertNotNull(om); + } +} From 6f8b58a1d6cd1991dedf1b3a9c3542e6a1d414f1 Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Sun, 7 Jan 2024 12:18:01 -0600 Subject: [PATCH 04/12] all tests and quality gates met --- .../GlueSchemaRegistryConfiguration.java | 3 +- .../GlueSchemaRegistryConfigurationTest.java | 10 +---- .../utils/json/ObjectMapperUtils.java | 2 +- .../utils/json/ObjectMapperUtilsTest.java | 39 +++++++++++++++++-- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java index 5ed5a56a..b99f196c 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java @@ -330,13 +330,12 @@ private void validateAndSetJavaTimeModule(Map configs) { if (isPresent(configs, AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE)) { String moduleClassName = String.valueOf(configs.get(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE)); this.javaTimeModuleClass = moduleClassName; - loadJavaTimeModule(); } } public SimpleModule loadJavaTimeModule() { try { - Class moduleClass = Class.forName(this.javaTimeModuleClass); + Class moduleClass = Class.forName(this.getJavaTimeModuleClass()); return (SimpleModule) moduleClass.getConstructor().newInstance(); } catch (Exception e) { String message = String.format("Invalid JavaTimeModule specified: %s", this.javaTimeModuleClass); diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java index fca11387..48cc9628 100644 --- a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java @@ -407,19 +407,13 @@ public void testBuildConfig_invalidProxyUrl_throwsException() { assertEquals("Proxy URL property is not a valid URL: "+proxy, exception.getMessage()); } - @Test - public void testBuildConfig_defaultJavaTimeModule_succeeds() { - Properties props = createTestProperties(); - GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); - assertNull(glueSchemaRegistryConfiguration.loadJavaTimeModule()); - } - @Test public void testBuildConfig_javaTimeModuleEnabledWithoutDependency_throwException() { Properties props = createTestProperties(); String moduleClassName = "com.fasterxml.jackson.datatype.jsr310.JavaTimeModule"; props.put(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE, moduleClassName); - Exception exception = assertThrows(AWSSchemaRegistryException.class, () -> new GlueSchemaRegistryConfiguration(props)); + GlueSchemaRegistryConfiguration cfg = new GlueSchemaRegistryConfiguration(props); + Exception exception = assertThrows(AWSSchemaRegistryException.class, () -> cfg.loadJavaTimeModule()); String message = String.format("Invalid JavaTimeModule specified: %s", moduleClassName); assertEquals(message, exception.getMessage()); } diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java index 85f79f0c..be517c13 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java @@ -22,8 +22,8 @@ public static ObjectMapper create(GlueSchemaRegistryConfiguration configs) { .forEach(objectMapper::enable); } if (configs.getJavaTimeModuleClass() != null) { - objectMapper.registerModule(configs.loadJavaTimeModule()); objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + objectMapper.registerModule(configs.loadJavaTimeModule()); } } diff --git a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtilsTest.java b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtilsTest.java index 14f95b20..96e47fec 100644 --- a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtilsTest.java +++ b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtilsTest.java @@ -1,30 +1,61 @@ package com.amazonaws.services.schemaregistry.utils.json; import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import com.amazonaws.services.schemaregistry.exception.AWSSchemaRegistryException; import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; import org.junit.jupiter.api.Test; import java.util.HashMap; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; /** * Unit tests for testing object mapper */ public class ObjectMapperUtilsTest { - static GlueSchemaRegistryConfiguration testConfigs(Map configs) { - configs.putIfAbsent(AWSSchemaRegistryConstants.AWS_REGION, "US-West-1"); - return new GlueSchemaRegistryConfiguration(configs); + static GlueSchemaRegistryConfiguration testConfigs(Map map) { + return new GlueSchemaRegistryConfiguration(testMap(map)); + } + + static Map testMap(Map map) { + if (map == null) { + map = new HashMap<>(); + } + map.putIfAbsent(AWSSchemaRegistryConstants.AWS_REGION, "US-West-1"); + return map; } @Test void testCreate_succeeds() { - GlueSchemaRegistryConfiguration cfg = testConfigs(new HashMap()); + GlueSchemaRegistryConfiguration cfg = testConfigs(null); ObjectMapper om = ObjectMapperUtils.create(cfg); assertNotNull(om); } + + @Test + void testCreate_useJavaTimeModuleWithoutDeps_throwsException() { + Map map = testMap(null); + map.put(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE, "com.fasterxml.jackson.datatype.jsr310.JavaTimeModule"); + GlueSchemaRegistryConfiguration cfg = testConfigs(map); + assertThrows(AWSSchemaRegistryException.class, () -> ObjectMapperUtils.create(cfg)); + } + + @Test + void testCreate_useJavaTimeModuleWithMockTimeModule_succeeds() { + Map map = testMap(null); + map.put(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE, "com.fasterxml.jackson.datatype.jsr310.JavaTimeModule"); + + GlueSchemaRegistryConfiguration spy = spy(testConfigs(map)); + doReturn(new SimpleModule()).when(spy).loadJavaTimeModule(); + + ObjectMapper om = ObjectMapperUtils.create(spy); + assertNotNull(om); + } } From 983e50db06c069ed639f49b826421b30003cb7f0 Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Sun, 7 Jan 2024 12:31:49 -0600 Subject: [PATCH 05/12] updated readme docs for using Java 8 Dates in JSON --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index ec17d63d..5d93e8b7 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,26 @@ The recommended way to use the AWS Glue Schema Registry Library for Java is to c ``` +### Jackson Support for Java 8 Date Types in JSON + +To support Java 8 Dates in JSON add the [Jackson JSR310 Datatype](https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310) dependency to implementing project class path. + +For example, in a Maven based project include the latest dependency. + +```xml + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + 2.16.1 + +``` + +Then add the following configuration property to specify the fully qualified class path to the JavaTimeModule like so. + +```java +properties.put(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE, "com.fasterxml.jackson.datatype.jsr310.JavaTimeModule"); +``` + #### Producer for Kafka with PROTOBUF format ```java From 217deabbf0f5dd8a00e54db56246d4e719bac83b Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Tue, 9 Jan 2024 00:08:18 -0600 Subject: [PATCH 06/12] add extensible objectmapperfactory --- .../GlueSchemaRegistryConfiguration.java | 7 ++++ .../utils/AWSSchemaRegistryConstants.java | 6 ++++ .../json/DefaultObjectMapperFactory.java | 33 +++++++++++++++++++ .../utils/json/ObjectMapperFactory.java | 9 +++++ .../utils/json/ObjectMapperUtils.java | 30 +++++------------ .../json/JsonDeserializerTest.java | 3 +- 6 files changed, 66 insertions(+), 22 deletions(-) create mode 100644 serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/DefaultObjectMapperFactory.java create mode 100644 serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperFactory.java diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java index b99f196c..6d14949b 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java @@ -60,6 +60,7 @@ public class GlueSchemaRegistryConfiguration { private String secondaryDeserializer; private URI proxyUrl; private String javaTimeModuleClass; + private String objectMapperFactory = "com.amazonaws.services.schemaregistry.utils.json.DefaultObjectMapperFactory"; /** * Name of the application using the serializer/deserializer. @@ -333,6 +334,12 @@ private void validateAndSetJavaTimeModule(Map configs) { } } + private void validateAndSetObjectMapperFactory(Map configs) { + if (isPresent(configs, AWSSchemaRegistryConstants.OBJECT_MAPPER_FACTORY)) { + this.objectMapperFactory = String.valueOf(configs.get(AWSSchemaRegistryConstants.OBJECT_MAPPER_FACTORY)); + } + } + public SimpleModule loadJavaTimeModule() { try { Class moduleClass = Class.forName(this.getJavaTimeModuleClass()); diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/utils/AWSSchemaRegistryConstants.java b/common/src/main/java/com/amazonaws/services/schemaregistry/utils/AWSSchemaRegistryConstants.java index a5867709..3c63018a 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/utils/AWSSchemaRegistryConstants.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/utils/AWSSchemaRegistryConstants.java @@ -179,6 +179,12 @@ public final class AWSSchemaRegistryConstants { */ public static final String REGISTER_JAVA_TIME_MODULE = "registerJavaTimeModule"; + /** + * Factory for creating custom Jackson ObjectMapper instances for use in handling JSON. + * Default: com.amazonaws.services.schemaregistry.utils.json.DefaultObjectMapperFactory + */ + public static final String OBJECT_MAPPER_FACTORY = "objectMapperFactory"; + /** * Private constructor to avoid initialization of the class. */ diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/DefaultObjectMapperFactory.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/DefaultObjectMapperFactory.java new file mode 100644 index 00000000..c36b9a97 --- /dev/null +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/DefaultObjectMapperFactory.java @@ -0,0 +1,33 @@ +package com.amazonaws.services.schemaregistry.utils.json; + +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import org.apache.commons.collections4.CollectionUtils; + +public class DefaultObjectMapperFactory implements ObjectMapperFactory { + + @Override + public ObjectMapper newInstance(GlueSchemaRegistryConfiguration configs) { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setNodeFactory(JsonNodeFactory.withExactBigDecimals(true)); + + if (configs != null) { + if (!CollectionUtils.isEmpty(configs.getJacksonSerializationFeatures())) { + configs.getJacksonSerializationFeatures() + .forEach(objectMapper::enable); + } + if (!CollectionUtils.isEmpty(configs.getJacksonDeserializationFeatures())) { + configs.getJacksonDeserializationFeatures() + .forEach(objectMapper::enable); + } + if (configs.getJavaTimeModuleClass() != null) { + objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + objectMapper.registerModule(configs.loadJavaTimeModule()); + } + } + + return objectMapper; + } +} diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperFactory.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperFactory.java new file mode 100644 index 00000000..9e5b597a --- /dev/null +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperFactory.java @@ -0,0 +1,9 @@ +package com.amazonaws.services.schemaregistry.utils.json; + +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import com.fasterxml.jackson.databind.ObjectMapper; + +public interface ObjectMapperFactory { + + ObjectMapper newInstance(GlueSchemaRegistryConfiguration configs); +} diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java index be517c13..2628b7d9 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/utils/json/ObjectMapperUtils.java @@ -1,32 +1,20 @@ package com.amazonaws.services.schemaregistry.utils.json; import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import com.amazonaws.services.schemaregistry.exception.AWSSchemaRegistryException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import org.apache.commons.collections4.CollectionUtils; public class ObjectMapperUtils { public static ObjectMapper create(GlueSchemaRegistryConfiguration configs) { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.setNodeFactory(JsonNodeFactory.withExactBigDecimals(true)); - - if (configs != null) { - if (!CollectionUtils.isEmpty(configs.getJacksonSerializationFeatures())) { - configs.getJacksonSerializationFeatures() - .forEach(objectMapper::enable); - } - if (!CollectionUtils.isEmpty(configs.getJacksonDeserializationFeatures())) { - configs.getJacksonDeserializationFeatures() - .forEach(objectMapper::enable); - } - if (configs.getJavaTimeModuleClass() != null) { - objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - objectMapper.registerModule(configs.loadJavaTimeModule()); - } + try { + Class moduleClass = Class.forName(configs.getObjectMapperFactory()); + ObjectMapperFactory factory = (ObjectMapperFactory) moduleClass.getConstructor().newInstance(); + return factory.newInstance(configs); + } catch (Exception e) { + String message = String.format("Failed to instantiate ObjectMapperFactory: %s", + configs.getObjectMapperFactory()); + throw new AWSSchemaRegistryException(message, e); } - - return objectMapper; } } diff --git a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializerTest.java b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializerTest.java index 14168ff0..411e132b 100644 --- a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializerTest.java +++ b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/json/JsonDeserializerTest.java @@ -14,6 +14,7 @@ */ package com.amazonaws.services.schemaregistry.deserializers.json; +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; import org.junit.jupiter.api.Test; import java.nio.ByteBuffer; @@ -26,10 +27,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; public class JsonDeserializerTest { - private JsonDeserializer defaultJsonDeserializer = new JsonDeserializer(null); @Test public void testDeserialize_nullArgs_throwsException() { + JsonDeserializer defaultJsonDeserializer = new JsonDeserializer(new GlueSchemaRegistryConfiguration("us-west-1")); String testSchemaDefinition = "{\"$id\":\"https://example.com/geographical-location.schema.json\"," + "\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Longitude " + "and Latitude Values\",\"description\":\"A geographical coordinate.\"," From 26c519e07b6629e50f3c441853c735e2a4c4ea12 Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Wed, 10 Jan 2024 23:25:09 -0600 Subject: [PATCH 07/12] use objectmapper in jsonvalidator as well --- .../kafkaconnect/jsonschema/FromConnectTest.java | 2 +- .../schemaregistry/serializers/json/JsonSerializer.java | 7 ++++--- .../schemaregistry/serializers/json/JsonValidator.java | 9 ++++++++- .../serializers/json/JsonValidatorTest.java | 3 ++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/jsonschema-kafkaconnect-converter/src/test/java/com/amazonaws/services/schemaregistry/kafkaconnect/jsonschema/FromConnectTest.java b/jsonschema-kafkaconnect-converter/src/test/java/com/amazonaws/services/schemaregistry/kafkaconnect/jsonschema/FromConnectTest.java index d3782e1b..959537eb 100644 --- a/jsonschema-kafkaconnect-converter/src/test/java/com/amazonaws/services/schemaregistry/kafkaconnect/jsonschema/FromConnectTest.java +++ b/jsonschema-kafkaconnect-converter/src/test/java/com/amazonaws/services/schemaregistry/kafkaconnect/jsonschema/FromConnectTest.java @@ -57,7 +57,7 @@ public class FromConnectTest { private static final JsonNodeFactory JSON_NODE_FACTORY = TypeConverter.JSON_NODE_FACTORY; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final JsonValidator JSON_VALIDATOR = new JsonValidator(); + private static final JsonValidator JSON_VALIDATOR = new JsonValidator(OBJECT_MAPPER); private ConnectSchemaToJsonSchemaConverter connectSchemaToJsonSchemaConverter; private ConnectValueToJsonNodeConverter connectValueToJsonNodeConverter; diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java index c945e51a..e1bccdc3 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonSerializer.java @@ -35,7 +35,7 @@ */ @Slf4j public class JsonSerializer implements GlueSchemaRegistryDataFormatSerializer { - private static final JsonValidator JSON_VALIDATOR = new JsonValidator(); + private final JsonValidator jsonValidator; private final JsonSchemaGenerator jsonSchemaGenerator; private final ObjectMapper objectMapper; @Getter @@ -51,6 +51,7 @@ public class JsonSerializer implements GlueSchemaRegistryDataFormatSerializer { public JsonSerializer(GlueSchemaRegistryConfiguration configs) { this.schemaRegistrySerDeConfigs = configs; this.objectMapper = ObjectMapperUtils.create(configs); + this.jsonValidator = new JsonValidator(this.objectMapper); this.jsonSchemaGenerator = new JsonSchemaGenerator(this.objectMapper); } @@ -67,7 +68,7 @@ public byte[] serialize(Object data) { final JsonNode dataNode = getDataNode(data); final JsonNode schemaNode = getSchemaNode(data); - JSON_VALIDATOR.validateDataWithSchema(schemaNode, dataNode); + jsonValidator.validateDataWithSchema(schemaNode, dataNode); bytes = writeBytes(dataNode); return bytes; @@ -169,6 +170,6 @@ public void validate(String schemaDefinition, byte[] data) { public void validate(Object jsonDataWithSchema) { JsonNode schemaNode = getSchemaNode(jsonDataWithSchema); JsonNode dataNode = getDataNode(jsonDataWithSchema); - JSON_VALIDATOR.validateDataWithSchema(schemaNode, dataNode); + jsonValidator.validateDataWithSchema(schemaNode, dataNode); } } diff --git a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonValidator.java b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonValidator.java index 8b59c907..80dfbea9 100644 --- a/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonValidator.java +++ b/serializer-deserializer/src/main/java/com/amazonaws/services/schemaregistry/serializers/json/JsonValidator.java @@ -30,6 +30,13 @@ * Json validator */ public class JsonValidator { + + private final ObjectMapper mapper; + + public JsonValidator(ObjectMapper mapper) { + this.mapper = mapper; + } + /** * Validates data against JsonSchema * @param schemaNode @@ -37,7 +44,7 @@ public class JsonValidator { */ public void validateDataWithSchema(JsonNode schemaNode, JsonNode dataNode) { try { - ObjectMapper mapper = new ObjectMapper(); +// ObjectMapper mapper = new ObjectMapper(); JSONObject rawSchema = new JSONObject(mapper.writeValueAsString(schemaNode)); Schema schema = SchemaLoader.load(rawSchema, new ReferenceDisabledSchemaClient()); diff --git a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/serializers/json/JsonValidatorTest.java b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/serializers/json/JsonValidatorTest.java index 4a686a9a..5774ead3 100644 --- a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/serializers/json/JsonValidatorTest.java +++ b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/serializers/json/JsonValidatorTest.java @@ -13,8 +13,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; public class JsonValidatorTest { - private JsonValidator validator = new JsonValidator(); private ObjectMapper mapper = new ObjectMapper(); + private JsonValidator validator = new JsonValidator(mapper); + private String stringSchema = "{\n" + " \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n" + " \"description\": \"String schema\",\n" From d41e2c7d614a46b31d79d885057a7245ddc2b974 Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Thu, 11 Jan 2024 10:30:48 -0600 Subject: [PATCH 08/12] forgot to assign the objectmapperfactory override --- .../common/configs/GlueSchemaRegistryConfiguration.java | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java index 6d14949b..af582b1c 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java @@ -108,6 +108,7 @@ private void buildSchemaRegistryConfigs(Map configs) { validateAndSetSecondaryDeserializer(configs); validateAndSetProxyUrl(configs); validateAndSetJavaTimeModule(configs); + validateAndSetObjectMapperFactory(configs); } private void validateAndSetSecondaryDeserializer(Map configs) { From 42dba8f59999bcab9781be0611d13a8fd3780b65 Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Thu, 11 Jan 2024 11:55:54 -0600 Subject: [PATCH 09/12] more tests for objectmapperfactory --- .../GlueSchemaRegistryConfigurationTest.java | 18 ++++++++++++++++++ .../common/configs/TestableSimpleModule.java | 6 ++++++ 2 files changed, 24 insertions(+) create mode 100644 common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/TestableSimpleModule.java diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java index 48cc9628..dd38c1e6 100644 --- a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java @@ -407,6 +407,15 @@ public void testBuildConfig_invalidProxyUrl_throwsException() { assertEquals("Proxy URL property is not a valid URL: "+proxy, exception.getMessage()); } + @Test + public void testBuildConfig_javaTimeModuleEnabledMock_succeeds() { + Properties props = createTestProperties(); + String moduleClassName = "com.amazonaws.services.schemaregistry.common.configs.TestableSimpleModule"; + props.put(AWSSchemaRegistryConstants.REGISTER_JAVA_TIME_MODULE, moduleClassName); + GlueSchemaRegistryConfiguration cfg = new GlueSchemaRegistryConfiguration(props); + assertNotNull(cfg.loadJavaTimeModule()); + } + @Test public void testBuildConfig_javaTimeModuleEnabledWithoutDependency_throwException() { Properties props = createTestProperties(); @@ -417,4 +426,13 @@ public void testBuildConfig_javaTimeModuleEnabledWithoutDependency_throwExceptio String message = String.format("Invalid JavaTimeModule specified: %s", moduleClassName); assertEquals(message, exception.getMessage()); } + + @Test + public void testBuildConfig_specifyObjectMapperFactory_succeeds() { + Properties props = createTestProperties(); + String factoryClass = "com.amazonaws.services.schemaregistry.common.configs.TestableSimpleModule"; // could be any string + props.put(AWSSchemaRegistryConstants.OBJECT_MAPPER_FACTORY, factoryClass); + GlueSchemaRegistryConfiguration cfg = new GlueSchemaRegistryConfiguration(props); + assertEquals(factoryClass, cfg.getObjectMapperFactory()); + } } \ No newline at end of file diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/TestableSimpleModule.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/TestableSimpleModule.java new file mode 100644 index 00000000..fbc68016 --- /dev/null +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/TestableSimpleModule.java @@ -0,0 +1,6 @@ +package com.amazonaws.services.schemaregistry.common.configs; + +import com.fasterxml.jackson.databind.module.SimpleModule; + +public class TestableSimpleModule extends SimpleModule { +} From 2592ff2000b6d5d7fd28a7ca024fca103642b8a4 Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Thu, 11 Jan 2024 12:16:32 -0600 Subject: [PATCH 10/12] fix some other coverage issues --- .../GlueSchemaRegistryConfigurationTest.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java index dd38c1e6..471e67a4 100644 --- a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java @@ -18,6 +18,8 @@ import com.amazonaws.services.schemaregistry.exception.AWSSchemaRegistryException; import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; import com.amazonaws.services.schemaregistry.utils.AvroRecordType; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.SerializationFeature; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -435,4 +437,26 @@ public void testBuildConfig_specifyObjectMapperFactory_succeeds() { GlueSchemaRegistryConfiguration cfg = new GlueSchemaRegistryConfiguration(props); assertEquals(factoryClass, cfg.getObjectMapperFactory()); } + + @Test + public void testBuildConfig_jacksonSerializationFeatures_succeeds() { + Properties props = createTestProperties(); + List features = new ArrayList<>(); + features.add(SerializationFeature.INDENT_OUTPUT.toString()); + features.add(SerializationFeature.WRAP_EXCEPTIONS.toString()); + props.put(AWSSchemaRegistryConstants.JACKSON_SERIALIZATION_FEATURES, features); + GlueSchemaRegistryConfiguration cfg = new GlueSchemaRegistryConfiguration(props); + assertEquals(cfg.getJacksonSerializationFeatures().size(), 2); + } + + @Test + public void testBuildConfig_jacksonDeserializationFeatures_succeeds() { + Properties props = createTestProperties(); + List features = new ArrayList<>(); + features.add(DeserializationFeature.WRAP_EXCEPTIONS.toString()); + features.add(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES.toString()); + props.put(AWSSchemaRegistryConstants.JACKSON_DESERIALIZATION_FEATURES, features); + GlueSchemaRegistryConfiguration cfg = new GlueSchemaRegistryConfiguration(props); + assertEquals(cfg.getJacksonDeserializationFeatures().size(), 2); + } } \ No newline at end of file From ff2dd72ab217e92bbd8ae0614654db11d083d66b Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Thu, 11 Jan 2024 15:07:25 -0600 Subject: [PATCH 11/12] improved test coverage --- .../common/AWSSchemaRegistryClientTest.java | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java index c6a221ae..5abfcd7b 100644 --- a/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java @@ -32,25 +32,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.CreateSchemaRequest; -import software.amazon.awssdk.services.glue.model.CreateSchemaResponse; -import software.amazon.awssdk.services.glue.model.DataFormat; -import software.amazon.awssdk.services.glue.model.EntityNotFoundException; -import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionRequest; -import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionResponse; -import software.amazon.awssdk.services.glue.model.GetSchemaVersionRequest; -import software.amazon.awssdk.services.glue.model.GetSchemaVersionResponse; -import software.amazon.awssdk.services.glue.model.GetTagsRequest; -import software.amazon.awssdk.services.glue.model.GetTagsResponse; -import software.amazon.awssdk.services.glue.model.MetadataKeyValuePair; -import software.amazon.awssdk.services.glue.model.PutSchemaVersionMetadataRequest; -import software.amazon.awssdk.services.glue.model.PutSchemaVersionMetadataResponse; -import software.amazon.awssdk.services.glue.model.QuerySchemaVersionMetadataRequest; -import software.amazon.awssdk.services.glue.model.QuerySchemaVersionMetadataResponse; -import software.amazon.awssdk.services.glue.model.RegisterSchemaVersionRequest; -import software.amazon.awssdk.services.glue.model.RegisterSchemaVersionResponse; -import software.amazon.awssdk.services.glue.model.RegistryId; -import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.glue.model.*; import java.io.File; import java.io.IOException; @@ -693,6 +675,18 @@ public void testQuerySchemaTags_clientThrowsException_throwsException() throws N assertEquals(expectedMsg, awsSchemaRegistryException.getMessage()); } + @Test + public void testCreateSchema_existingSchema_throwsAlreadyExistsException() throws NoSuchFieldException, IllegalAccessException { + String testSchemaName = "test-schema"; + String testSchemaDefinition = "test-schema-definition"; + awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, + glueSchemaRegistryConfiguration); + + when(mockGlueClient.createSchema(any(CreateSchemaRequest.class))).thenThrow(AlreadyExistsException.create("Already exists", null)); + + assertThrows(AWSSchemaRegistryException.class, () -> awsSchemaRegistryClient.createSchema(testSchemaName, DataFormat.AVRO.toString(), testSchemaDefinition, getMetadata())); + } + private Map getMetadata() { Map metadata = new HashMap<>(); metadata.put("event-source-1", "topic1"); From 532b4e5be71e8cc332f34439ff892d46072014b7 Mon Sep 17 00:00:00 2001 From: Adam McQuistan Date: Thu, 11 Jan 2024 22:20:44 -0600 Subject: [PATCH 12/12] improve unittest coverage --- .../common/AWSSchemaRegistryClientTest.java | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java index 5abfcd7b..4c06c896 100644 --- a/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java @@ -50,10 +50,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) public class AWSSchemaRegistryClientTest { @@ -682,9 +679,22 @@ public void testCreateSchema_existingSchema_throwsAlreadyExistsException() throw awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); - when(mockGlueClient.createSchema(any(CreateSchemaRequest.class))).thenThrow(AlreadyExistsException.create("Already exists", null)); - - assertThrows(AWSSchemaRegistryException.class, () -> awsSchemaRegistryClient.createSchema(testSchemaName, DataFormat.AVRO.toString(), testSchemaDefinition, getMetadata())); + AWSSchemaRegistryClient awsSchemaRegistryClientSpy = spy(awsSchemaRegistryClient); + AlreadyExistsException ex = AlreadyExistsException.builder().message("Already exists").build(); + when(mockGlueClient.createSchema(any(CreateSchemaRequest.class))).thenThrow(ex); + + UUID expectedId = UUID.randomUUID(); + String expectedDataFormat = DataFormat.AVRO.toString(); + Map expectedMetadata = getMetadata(); + doReturn(expectedId).when(awsSchemaRegistryClientSpy).registerSchemaVersion( + testSchemaDefinition, + testSchemaName, + expectedDataFormat, + expectedMetadata + ); + + UUID actualId = awsSchemaRegistryClientSpy.createSchema(testSchemaName, expectedDataFormat, testSchemaDefinition, expectedMetadata); + assertEquals(expectedId, actualId); } private Map getMetadata() {