diff --git a/.gitignore b/.gitignore index d9773299..ae8a3b4e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ **/*.project **/*.classpath **/*.factorypath -**/dependency-reduced-pom.xml \ No newline at end of file +**/dependency-reduced-pom.xml +/integration-tests/* \ No newline at end of file diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClient.java b/common/src/main/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClient.java index 949e6563..fe78a7fc 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClient.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClient.java @@ -37,16 +37,22 @@ import software.amazon.awssdk.services.glue.GlueClient; import software.amazon.awssdk.services.glue.GlueClientBuilder; import software.amazon.awssdk.services.glue.model.AlreadyExistsException; +import software.amazon.awssdk.services.glue.model.Compatibility; import software.amazon.awssdk.services.glue.model.CreateSchemaRequest; import software.amazon.awssdk.services.glue.model.CreateSchemaResponse; import software.amazon.awssdk.services.glue.model.DataFormat; import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionRequest; import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionResponse; +import software.amazon.awssdk.services.glue.model.GetSchemaRequest; +import software.amazon.awssdk.services.glue.model.GetSchemaResponse; import software.amazon.awssdk.services.glue.model.GetSchemaVersionRequest; import software.amazon.awssdk.services.glue.model.GetSchemaVersionResponse; import software.amazon.awssdk.services.glue.model.GetTagsRequest; import software.amazon.awssdk.services.glue.model.GetTagsResponse; import software.amazon.awssdk.services.glue.model.GlueRequest; +import software.amazon.awssdk.services.glue.model.ListSchemaVersionsRequest; +import software.amazon.awssdk.services.glue.model.ListSchemaVersionsResponse; +import software.amazon.awssdk.services.glue.model.MetadataInfo; import software.amazon.awssdk.services.glue.model.MetadataKeyValuePair; import software.amazon.awssdk.services.glue.model.PutSchemaVersionMetadataRequest; import software.amazon.awssdk.services.glue.model.PutSchemaVersionMetadataResponse; @@ -56,12 +62,20 @@ import software.amazon.awssdk.services.glue.model.RegisterSchemaVersionResponse; import software.amazon.awssdk.services.glue.model.RegistryId; import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.glue.model.SchemaVersionListItem; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.StringJoiner; import java.util.UUID; +import java.util.regex.Pattern; /** * Handles all the requests related to the schema management. @@ -180,7 +194,7 @@ public GetSchemaVersionResponse getSchemaVersionResponse(@NonNull String schemaV return schemaVersionResponse; } - private GetSchemaVersionRequest getSchemaVersionRequest(String schemaVersionId) { + public GetSchemaVersionRequest getSchemaVersionRequest(String schemaVersionId) { GetSchemaVersionRequest getSchemaVersionRequest = GetSchemaVersionRequest.builder() .schemaVersionId(schemaVersionId).build(); return getSchemaVersionRequest; @@ -193,6 +207,35 @@ private void validateSchemaVersionResponse(GetSchemaVersionResponse schemaVersio } } + public GetSchemaResponse getSchemaResponse(@NonNull SchemaId schemaId) + throws AWSSchemaRegistryException { + GetSchemaResponse schemaResponse = null; + + try { + schemaResponse = client.getSchema(getSchemaRequest(schemaId)); + validateSchemaResponse(schemaResponse, schemaId); + } catch (Exception e) { + String errorMessage = String.format("Failed to get schema Id = %s", schemaId); + throw new AWSSchemaRegistryException(errorMessage, e); + } + + return schemaResponse; + } + + private GetSchemaRequest getSchemaRequest(SchemaId schemaId) { + GetSchemaRequest getSchemaRequest = GetSchemaRequest.builder() + .schemaId(schemaId) + .build(); + return getSchemaRequest; + } + + private void validateSchemaResponse(GetSchemaResponse schemaResponse, SchemaId schemaId) { + if (schemaResponse == null) { + String message = String.format("Schema is not present for the schema id = %s", schemaId); + throw new AWSSchemaRegistryException(message); + } + } + private UUID returnSchemaVersionIdIfAvailable(GetSchemaByDefinitionResponse response) { if (response.schemaVersionId() != null && response.statusAsString().equals(AWSSchemaRegistryConstants.SchemaVersionStatus.AVAILABLE.toString())) { @@ -266,6 +309,140 @@ public UUID createSchema(String schemaName, return schemaVersionId; } + /** + * Create a schema in target registry using the Glue client and register the schema versions found in the source schema + * to the target schema in the same order. Also used to cache the schemas along with the version ids. + * @param schemaName Schema Name + * @param dataFormat Data Format + * @param schemaDefinition Schema Definition + * @param compatibility Schema Compatibility mode + * @param metadata schema version metadata + * @return Map of SchemaV2 with VersionIds + * @throws AWSSchemaRegistryException on any error during the schema creation + */ + //TODO: Remove this after importing the tests to converter + public Map createSchemaAndRegisterAllSchemaVersions(String schemaName, + String dataFormat, + String schemaDefinition, + Compatibility compatibility, + Map metadata) throws AWSSchemaRegistryException { + Map schemaWithVersionId = new HashMap<>(); + UUID schemaVersionId; + + try { + //Get list of all schema versions + List schemaVersionList = getSchemaVersions(schemaName, 50); + + for (int idx = 0; idx < schemaVersionList.size(); idx++){ + //Get details of each schema versions + GetSchemaVersionResponse getSchemaVersionResponse = + client.getSchemaVersion(getSchemaVersionRequest( + schemaVersionList.get(idx).schemaVersionId())); + + String schemaNameFromArn = getSchemaNameFromArn(schemaVersionList.get(idx).schemaArn()); + + //Get the metadata information for each version + QuerySchemaVersionMetadataResponse querySchemaVersionMetadataResponse = querySourceSchemaVersionMetadata(UUID.fromString(getSchemaVersionResponse.schemaVersionId())); + Map metadataInfo = getMetadataInfo(querySchemaVersionMetadataResponse.metadataInfoMap()); + + //Create the schema with the first schema version + if (idx == 0) { + log.info("Auto Creating schema with schemaName: {} and schemaDefinition : {}", + schemaNameFromArn, getSchemaVersionResponse.schemaDefinition()); + + //Create the schema + CreateSchemaResponse createSchemaResponse = client.createSchema(getCreateSchemaRequestObjectV2( + schemaNameFromArn, + getSchemaVersionResponse.dataFormat().toString(), + getSchemaVersionResponse.schemaDefinition(), compatibility)); + schemaVersionId = UUID.fromString(createSchemaResponse.schemaVersionId()); + + //Add version metadata to the schema version + putSchemaVersionMetadata(schemaVersionId, metadataInfo); + } else { + //Register subsequent schema versions + schemaVersionId = registerSchemaVersion(getSchemaVersionResponse.schemaDefinition(), + schemaNameFromArn, getSchemaVersionResponse.dataFormat().toString(), metadataInfo); + } + + Schema schema = new Schema(getSchemaVersionResponse.schemaDefinition(), getSchemaVersionResponse.dataFormat().toString(), schemaNameFromArn); + + //Create a map of schema and schemaVersionId + schemaWithVersionId.put(schema, schemaVersionId); + } + } catch (AlreadyExistsException e) { + log.warn("Schema is already created, this could be caused by multiple producers racing to " + + "auto-create schema."); + schemaVersionId = registerSchemaVersion(schemaDefinition, schemaName, dataFormat, metadata); + Schema schema = new Schema(schemaDefinition, dataFormat, schemaName); + schemaWithVersionId.put(schema, schemaVersionId); + putSchemaVersionMetadata(schemaVersionId, metadata); + + } catch (Exception e) { + String errorMessage = String.format( + "Create schema :: Call failed when creating the schema with the schema registry for" + + " schema name = %s", schemaName); + throw new AWSSchemaRegistryException(errorMessage, e); + } + + return schemaWithVersionId; + } + + public List getSchemaVersions(String schemaName, Integer replicateSchemaVersionCount) { + ListSchemaVersionsRequest listSchemaVersionsRequest = getListSchemaVersionsRequest(schemaName); + List schemaVersionList = new ArrayList<>(); + boolean done = false; + + while(!done) { + //Get list of schema versions from source registry + ListSchemaVersionsResponse listSchemaVersionsResponse = client.listSchemaVersions(listSchemaVersionsRequest); + schemaVersionList = listSchemaVersionsResponse.schemas(); + + //Keep paginating till the end + if (listSchemaVersionsResponse.nextToken() == null) { + done = true; + } + + //Create the request object to get next set of results using the nextToken + listSchemaVersionsRequest = getListSchemaVersionsRequest(schemaName, listSchemaVersionsResponse.nextToken()); + } + + //Copy the schemaVersionList to a new list as the existing list is not modifiable. + List modifiableSchemaVersionList = new ArrayList(schemaVersionList); + + //Sort the schemaVersionList based on versionNumber in ascending order. + //This is important as the item in the list are in random order + //and we need to maintain the ordering of versions + Collections.sort(modifiableSchemaVersionList, Comparator.comparing(SchemaVersionListItem::versionNumber)); + + //int replicateSchemaVersionCount = glueSchemaRegistryConfiguration.getReplicateSchemaVersionCount(); + + //Get the list of schema versions equal to the replicateSchemaVersionCount + //If the list is smaller than replicateSchemaVersionCount, return the whole list. + modifiableSchemaVersionList = modifiableSchemaVersionList.subList(0, + Math.min(replicateSchemaVersionCount, modifiableSchemaVersionList.size())); + + return modifiableSchemaVersionList; + } + + //TODO: Remove this + private String getSchemaNameFromArn(String schemaArn) { + String[] tokens = schemaArn.split(Pattern.quote("/")); + return tokens[tokens.length - 1]; + } + + //TODO: Remove this + private Map getMetadataInfo(Map metadataInfoMap) { + Map metadata = new HashMap<>(); + Iterator> iterator = metadataInfoMap.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + metadata.put(entry.getKey(), entry.getValue().metadataValue()); + } + + return metadata; + } + /** * Register the schema and return schema version Id once it is available. * @param schemaDefinition Schema Definition @@ -298,7 +475,6 @@ public GetSchemaVersionResponse registerSchemaVersion(String schemaDefinition, S try { RegisterSchemaVersionResponse registerSchemaVersionResponse = client.registerSchemaVersion(getRegisterSchemaVersionRequest(schemaDefinition, schemaName)); - log.info("Registered the schema version with schema version id = {} and with version number = {} and " + "status {}", registerSchemaVersionResponse.schemaVersionId(), registerSchemaVersionResponse.versionNumber(), registerSchemaVersionResponse.statusAsString()); @@ -329,7 +505,7 @@ private GetSchemaVersionResponse transformToGetSchemaVersionResponse(RegisterSch .build(); } - private CreateSchemaRequest getCreateSchemaRequestObject(String schemaName, String dataFormat, String schemaDefinition) { + public CreateSchemaRequest getCreateSchemaRequestObject(String schemaName, String dataFormat, String schemaDefinition) { return CreateSchemaRequest .builder() .dataFormat(DataFormat.valueOf(dataFormat)) @@ -342,6 +518,35 @@ private CreateSchemaRequest getCreateSchemaRequestObject(String schemaName, Stri .build(); } + //TODO: Remove this + public CreateSchemaRequest getCreateSchemaRequestObjectV2(String schemaName, String dataFormat, String schemaDefinition, Compatibility compatibility) { + return CreateSchemaRequest + .builder() + .dataFormat(DataFormat.valueOf(dataFormat)) + .description(glueSchemaRegistryConfiguration.getDescription()) + .registryId(RegistryId.builder().registryName(glueSchemaRegistryConfiguration.getRegistryName()).build()) + .schemaName(schemaName) + .schemaDefinition(schemaDefinition) + .compatibility(compatibility) + .tags(glueSchemaRegistryConfiguration.getTags()) + .build(); + } + + private ListSchemaVersionsRequest getListSchemaVersionsRequest(String schemaName, String nextToken) { + return ListSchemaVersionsRequest + .builder() + .nextToken(nextToken) + .schemaId(SchemaId.builder().schemaName(schemaName).registryName(glueSchemaRegistryConfiguration.getRegistryName()).build()) + .build(); + } + + private ListSchemaVersionsRequest getListSchemaVersionsRequest(String schemaName) { + return ListSchemaVersionsRequest + .builder() + .schemaId(SchemaId.builder().schemaName(schemaName).registryName(glueSchemaRegistryConfiguration.getRegistryName()).build()) + .build(); + } + private RegisterSchemaVersionRequest getRegisterSchemaVersionRequest(String schemaDefinition, String schemaName) { return RegisterSchemaVersionRequest .builder() @@ -486,6 +691,26 @@ public QuerySchemaVersionMetadataResponse querySchemaVersionMetadata(UUID schema return response; } + /** + * Query metadata for schema version and return the response object + * + * @param schemaVersionId Schema Version Id + * @return QuerySchemaVersionMetadataResponse object + * @throws AWSSchemaRegistryException on any error during putting metadata + */ + public QuerySchemaVersionMetadataResponse querySourceSchemaVersionMetadata(UUID schemaVersionId) { + QuerySchemaVersionMetadataResponse response = null; + try { + response = client.querySchemaVersionMetadata(createQuerySchemaVersionMetadataRequest(schemaVersionId)); + } catch (Exception e) { + String errorMessage = String.format("Query schema version metadata :: Call failed when query metadata for schema version id = %s", + schemaVersionId.toString()); + throw new AWSSchemaRegistryException(errorMessage, e); + } + + return response; + } + private QuerySchemaVersionMetadataRequest createQuerySchemaVersionMetadataRequest(UUID schemaVersionId) { return QuerySchemaVersionMetadataRequest .builder() diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java index ce7d870d..e3649b07 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java @@ -198,13 +198,13 @@ private void validateAndSetAWSEndpoint(Map configs) { private void validateAndSetProxyUrl(Map configs) { if (isPresent(configs, AWSSchemaRegistryConstants.PROXY_URL)) { - String value = (String) configs.get(AWSSchemaRegistryConstants.PROXY_URL); - try { - this.proxyUrl = URI.create(value); - } catch (IllegalArgumentException e) { - String message = String.format("Proxy URL property is not a valid URL: %s", value); - throw new AWSSchemaRegistryException(message, e); - } + String value = (String) configs.get(AWSSchemaRegistryConstants.PROXY_URL); + try { + this.proxyUrl = URI.create(value); + } catch (IllegalArgumentException e) { + String message = String.format("Proxy URL property is not a valid URL: %s", value); + throw new AWSSchemaRegistryException(message, e); + } } } @@ -323,7 +323,7 @@ private void validateAndSetJacksonDeserializationFeatures(Map configs } } - private boolean isPresent(Map configs, + public boolean isPresent(Map configs, String key) { if (!GlueSchemaRegistryUtils.getInstance() .checkIfPresentInMap(configs, key)) { diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java index c6a221ae..987ad50c 100644 --- a/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClientTest.java @@ -23,25 +23,33 @@ import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.AlreadyExistsException; +import software.amazon.awssdk.services.glue.model.Compatibility; import software.amazon.awssdk.services.glue.model.CreateSchemaRequest; import software.amazon.awssdk.services.glue.model.CreateSchemaResponse; import software.amazon.awssdk.services.glue.model.DataFormat; import software.amazon.awssdk.services.glue.model.EntityNotFoundException; import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionRequest; import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionResponse; +import software.amazon.awssdk.services.glue.model.GetSchemaRequest; +import software.amazon.awssdk.services.glue.model.GetSchemaResponse; import software.amazon.awssdk.services.glue.model.GetSchemaVersionRequest; import software.amazon.awssdk.services.glue.model.GetSchemaVersionResponse; import software.amazon.awssdk.services.glue.model.GetTagsRequest; import software.amazon.awssdk.services.glue.model.GetTagsResponse; +import software.amazon.awssdk.services.glue.model.ListSchemaVersionsRequest; +import software.amazon.awssdk.services.glue.model.ListSchemaVersionsResponse; +import software.amazon.awssdk.services.glue.model.MetadataInfo; import software.amazon.awssdk.services.glue.model.MetadataKeyValuePair; import software.amazon.awssdk.services.glue.model.PutSchemaVersionMetadataRequest; import software.amazon.awssdk.services.glue.model.PutSchemaVersionMetadataResponse; @@ -51,6 +59,8 @@ import software.amazon.awssdk.services.glue.model.RegisterSchemaVersionResponse; import software.amazon.awssdk.services.glue.model.RegistryId; import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.glue.model.SchemaVersionListItem; +import software.amazon.awssdk.services.glue.model.SchemaVersionStatus; import java.io.File; import java.io.IOException; @@ -77,24 +87,32 @@ public class AWSSchemaRegistryClientTest { @Mock private GlueClient mockGlueClient; + @Mock + private GlueClient mockSourceRegistryGlueClient; private final Map configs = new HashMap<>(); private AWSSchemaRegistryClient awsSchemaRegistryClient; private GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration; private static String userSchemaDefinition; + private static String userSchemaDefinition2; private static GenericRecord genericUserAvroRecord; + private static GenericRecord genericUserAvroRecord2; private Schema schema = null; + private Schema schema2 = null; private Map testTags; private static final UUID SCHEMA_ID_FOR_TESTING = UUID.fromString("b7b4a7f0-9c96-4e4a-a687-fb5de9ef0c63"); + private static final UUID SCHEMA_ID_FOR_TESTING2 = UUID.fromString("310153e9-9a54-4b12-a513-a23fc543ed2f"); + private static final String SCHEMA_ARN_FOR_TESTING = "test-schema-arn"; public static final String AVRO_USER_SCHEMA_FILE = "src/test/java/resources/avro/user.avsc"; + public static final String AVRO_USER_SCHEMA_FILE2 = "src/test/java/resources/avro/user2.avsc"; - @BeforeEach public void setup() { awsSchemaRegistryClient = new AWSSchemaRegistryClient(mockGlueClient); Schema.Parser parser = new Schema.Parser(); try { schema = parser.parse(new File(AVRO_USER_SCHEMA_FILE)); + schema2 = parser.parse(new File(AVRO_USER_SCHEMA_FILE2)); } catch (IOException e) { fail("Catch IOException: ", e); } @@ -106,7 +124,16 @@ public void setup() { testTags = new HashMap<>(); testTags.put("testKey", "testValue"); + genericUserAvroRecord2 = new GenericData.Record(schema2); + genericUserAvroRecord2.put("name", "sansa"); + genericUserAvroRecord2.put("favorite_number", 99); + genericUserAvroRecord2.put("favorite_color", "red"); + genericUserAvroRecord2.put("gender", "MALE"); + testTags = new HashMap<>(); + testTags.put("testKey", "testValue"); + userSchemaDefinition = AVROUtils.getInstance().getSchemaDefinition(genericUserAvroRecord); + userSchemaDefinition2 = AVROUtils.getInstance().getSchemaDefinition(genericUserAvroRecord2); configs.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, "https://test"); configs.put(AWSSchemaRegistryConstants.AWS_REGION, "us-west-2"); @@ -291,6 +318,31 @@ public void testGetSchemaVersionResponse_setSchemaVersionId_returnsResponseSchem assertEquals(SCHEMA_ID_FOR_TESTING.toString(), awsSchemaRegistryClient.getSchemaVersionResponse(SCHEMA_ID_FOR_TESTING.toString()).schemaVersionId()); } + @Test + public void testGetSchemaResponse_nullSchemaId_throwsException() { + Assertions.assertThrows(IllegalArgumentException.class , () -> awsSchemaRegistryClient + .getSchemaResponse(null)); + } + + @Test + public void testGetSchemaResponse_setSchemaId_returnsSchemaResponse() { + GetSchemaResponse getSchemaResponse = GetSchemaResponse.builder().schemaArn(SCHEMA_ARN_FOR_TESTING).build(); + SchemaId schemaId = SchemaId.builder().schemaArn(SCHEMA_ARN_FOR_TESTING).build(); + GetSchemaRequest getSchemaRequest = GetSchemaRequest.builder().schemaId(schemaId).build(); + when(mockGlueClient.getSchema(getSchemaRequest)).thenReturn(getSchemaResponse); + + assertEquals(SCHEMA_ARN_FOR_TESTING, awsSchemaRegistryClient.getSchemaResponse(schemaId).schemaArn()); + } + + @Test + public void testGetSchemaResponse_nullSchemaResponse_throwsException() { + SchemaId schemaId = SchemaId.builder().schemaArn(SCHEMA_ARN_FOR_TESTING).build(); + GetSchemaRequest getSchemaRequest = GetSchemaRequest.builder().schemaId(schemaId).build(); + when(mockGlueClient.getSchema(getSchemaRequest)).thenReturn(null); + + Assertions.assertThrows(AWSSchemaRegistryException.class , () -> awsSchemaRegistryClient.getSchemaResponse(schemaId)); + } + private Map getConfigsWithAutoRegistrationSetting(boolean autoRegistrationSetting) { Map localConfigs = new HashMap<>(); @@ -375,6 +427,286 @@ public void testCreateSchema_clientExceptionResponse_returnsAWSSchemaRegistryExc } } + @Test + public void testcreateSchemaAndRegisterAllSchemaVersions_schemaNameWithDataFormat_returnsResponseSuccessfully() throws NoSuchFieldException, IllegalAccessException { + awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, + glueSchemaRegistryConfiguration); + Compatibility SCHEMA_COMPATIBILITY_MODE = Compatibility.FORWARD_ALL; + String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME).toString(); + String dataFormatName = DataFormat.AVRO.name(); + String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME).toString(); + Long schemaVersionNumber = 1L; + Long schemaVersionNumber2 = 2L; + + mockListSchemaVersions(schemaName, registryName, schemaVersionNumber, schemaVersionNumber2); + mockGetSchemaVersions(schemaVersionNumber, schemaVersionNumber2); + mockQuerySchemaVersionMetadata(); + mockCreateSchema(schemaName, dataFormatName, glueSchemaRegistryConfiguration); + mockRegisterSchemaVersion2(schemaName, registryName, schemaVersionNumber); + + Map schemaWithVersionId = awsSchemaRegistryClient + .createSchemaAndRegisterAllSchemaVersions(schemaName, dataFormatName, userSchemaDefinition, SCHEMA_COMPATIBILITY_MODE, getMetadata()); + + com.amazonaws.services.schemaregistry.common.Schema expectedSchema = new com.amazonaws.services.schemaregistry.common.Schema(userSchemaDefinition, dataFormatName, schemaName); + com.amazonaws.services.schemaregistry.common.Schema expectedSchema2 = new com.amazonaws.services.schemaregistry.common.Schema(userSchemaDefinition2, dataFormatName, schemaName); + + assertEquals(SCHEMA_ID_FOR_TESTING, schemaWithVersionId.get(expectedSchema)); + assertEquals(SCHEMA_ID_FOR_TESTING2, schemaWithVersionId.get(expectedSchema2)); + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testcreateSchemaAndRegisterAllSchemaVersions_clientExceptionResponse_returnsAlreadyExistsException() throws NoSuchFieldException, IllegalAccessException { + awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, + glueSchemaRegistryConfiguration); + Compatibility SCHEMA_COMPATIBILITY_MODE = Compatibility.FORWARD_ALL; + String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME).toString(); + String dataFormatName = DataFormat.AVRO.name(); + String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME).toString(); + Long schemaVersionNumber = 1L; + Long schemaVersionNumber2 = 2L; + + mockListSchemaVersions(schemaName, registryName, schemaVersionNumber, schemaVersionNumber2); + mockGetSchemaVersions(schemaVersionNumber, schemaVersionNumber2); + mockQuerySchemaVersionMetadata(); + mockCreateSchema(schemaName, dataFormatName,glueSchemaRegistryConfiguration); + mockRegisterSchemaVersion2(schemaName, registryName, schemaVersionNumber); + + awsSchemaRegistryClient.createSchemaAndRegisterAllSchemaVersions(schemaName, dataFormatName, userSchemaDefinition, SCHEMA_COMPATIBILITY_MODE, getMetadata()); + + try { + CreateSchemaRequest createSchemaRequest = CreateSchemaRequest.builder() + .dataFormat(DataFormat.AVRO) + .description(glueSchemaRegistryConfiguration.getDescription()) + .schemaName(schemaName) + .schemaDefinition(userSchemaDefinition) + .compatibility(SCHEMA_COMPATIBILITY_MODE) + .tags(glueSchemaRegistryConfiguration.getTags()) + .registryId(RegistryId.builder().registryName(glueSchemaRegistryConfiguration.getRegistryName()).build()) + .build(); + + when(mockGlueClient.createSchema(createSchemaRequest)).thenThrow(AlreadyExistsException.class); + + mockRegisterSchemaVersion(schemaName, registryName, schemaVersionNumber); + + awsSchemaRegistryClient.createSchemaAndRegisterAllSchemaVersions(schemaName, dataFormatName, userSchemaDefinition, SCHEMA_COMPATIBILITY_MODE, getMetadata()); + + } catch (Exception e) { + assertEquals(AlreadyExistsException.class, e.getCause().getClass()); + } + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + public void testcreateSchemaAndRegisterAllSchemaVersions_clientExceptionResponse_returnsAWSSchemaRegistryException() throws NoSuchFieldException, IllegalAccessException { + awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, + glueSchemaRegistryConfiguration); + Compatibility SCHEMA_COMPATIBILITY_MODE = Compatibility.FORWARD_ALL; + String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME).toString(); + String dataFormatName = DataFormat.AVRO.name(); + String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME).toString(); + Long schemaVersionNumber = 1L; + Long schemaVersionNumber2 = 2L; + + mockListSchemaVersions(schemaName, registryName, schemaVersionNumber, schemaVersionNumber2); + mockGetSchemaVersions(schemaVersionNumber, schemaVersionNumber2); + mockQuerySchemaVersionMetadata(); + + CreateSchemaRequest createSchemaRequest = CreateSchemaRequest.builder() + .dataFormat(DataFormat.AVRO) + .description(glueSchemaRegistryConfiguration.getDescription()) + .schemaName(schemaName) + .schemaDefinition(userSchemaDefinition) + .compatibility(SCHEMA_COMPATIBILITY_MODE) + .tags(glueSchemaRegistryConfiguration.getTags()) + .registryId(RegistryId.builder().registryName(glueSchemaRegistryConfiguration.getRegistryName()).build()) + .build(); + + when(mockGlueClient.createSchema(createSchemaRequest)).thenThrow(EntityNotFoundException.class); + + try { + awsSchemaRegistryClient.createSchemaAndRegisterAllSchemaVersions(schemaName, dataFormatName, userSchemaDefinition, SCHEMA_COMPATIBILITY_MODE, getMetadata()); + } catch (Exception e) { + assertEquals(EntityNotFoundException.class, e.getCause().getClass()); + assertEquals(AWSSchemaRegistryException.class, e.getClass()); + String expectedErrorMessage = "Create schema :: Call failed when creating the schema with the schema registry for schema name = " + schemaName; + assertEquals(expectedErrorMessage, e.getMessage()); + } + } + + @Test + public void testQuerySchemaVersionMetadata_returnsResponseSuccessfully() throws NoSuchFieldException, IllegalAccessException { + awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, + glueSchemaRegistryConfiguration); + + Map map = new HashMap<>(); + map.put("key", MetadataInfo.builder().metadataValue("value").build()); + + QuerySchemaVersionMetadataRequest querySchemaVersionMetadataRequest = QuerySchemaVersionMetadataRequest.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .build(); + + QuerySchemaVersionMetadataResponse querySchemaVersionMetadataResponse = QuerySchemaVersionMetadataResponse + .builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .metadataInfoMap(map) + .build(); + + when(mockGlueClient.querySchemaVersionMetadata(querySchemaVersionMetadataRequest)).thenReturn(querySchemaVersionMetadataResponse); + + QuerySchemaVersionMetadataResponse response = awsSchemaRegistryClient.querySchemaVersionMetadata(SCHEMA_ID_FOR_TESTING); + + assertEquals(SCHEMA_ID_FOR_TESTING.toString(), response.schemaVersionId()); + assertEquals(1, response.metadataInfoMap().size()); + assertEquals("value", response.metadataInfoMap().get("key").metadataValue()); + } + + @Test + public void testQuerySchemaVersionMetadata_returnsAWSSchemaRegistryException() throws NoSuchFieldException, IllegalAccessException { + awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, + glueSchemaRegistryConfiguration); + + QuerySchemaVersionMetadataRequest querySchemaVersionMetadataRequest = QuerySchemaVersionMetadataRequest.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .build(); + + AWSSchemaRegistryException awsSchemaRegistryException = new AWSSchemaRegistryException(); + + when(mockGlueClient.querySchemaVersionMetadata(querySchemaVersionMetadataRequest)).thenThrow(awsSchemaRegistryException); + + Exception exception = assertThrows(AWSSchemaRegistryException.class, + () -> awsSchemaRegistryClient.querySchemaVersionMetadata(SCHEMA_ID_FOR_TESTING)); + assertTrue( + exception.getMessage().contains(String.format("Query schema version metadata :: " + + "Call failed when query metadata for schema version id = %s", + SCHEMA_ID_FOR_TESTING))); + + } + + private void mockQuerySchemaVersionMetadata() { + QuerySchemaVersionMetadataRequest querySchemaVersionMetadataRequest = QuerySchemaVersionMetadataRequest.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .build(); + + QuerySchemaVersionMetadataResponse querySchemaVersionMetadataResponse = QuerySchemaVersionMetadataResponse + .builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .metadataInfoMap(new HashMap<>()) + .build(); + + QuerySchemaVersionMetadataRequest querySchemaVersionMetadataRequest2 = QuerySchemaVersionMetadataRequest.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) + .build(); + + QuerySchemaVersionMetadataResponse querySchemaVersionMetadataResponse2 = QuerySchemaVersionMetadataResponse + .builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) + .metadataInfoMap(new HashMap<>()) + .build(); + + when(mockSourceRegistryGlueClient.querySchemaVersionMetadata(querySchemaVersionMetadataRequest)).thenReturn(querySchemaVersionMetadataResponse); + when(mockSourceRegistryGlueClient.querySchemaVersionMetadata(querySchemaVersionMetadataRequest2)).thenReturn(querySchemaVersionMetadataResponse2); + } + + private void mockGetSchemaVersions(Long schemaVersionNumber, Long schemaVersionNumber2) { + GetSchemaVersionRequest getSchemaVersionRequest = GetSchemaVersionRequest.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()).build(); + + GetSchemaVersionResponse getSchemaVersionResponse = GetSchemaVersionResponse.builder() + .schemaDefinition(userSchemaDefinition) + .versionNumber(schemaVersionNumber) + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .dataFormat(DataFormat.AVRO) + .status(SchemaVersionStatus.AVAILABLE) + .build(); + + when(mockSourceRegistryGlueClient.getSchemaVersion(getSchemaVersionRequest)).thenReturn(getSchemaVersionResponse); + + GetSchemaVersionRequest getSchemaVersionRequest2 = GetSchemaVersionRequest.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()).build(); + + GetSchemaVersionResponse getSchemaVersionResponse2 = GetSchemaVersionResponse.builder() + .schemaDefinition(userSchemaDefinition2) + .versionNumber(schemaVersionNumber2) + .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) + .dataFormat(DataFormat.AVRO) + .status(SchemaVersionStatus.AVAILABLE) + .build(); + + when(mockSourceRegistryGlueClient.getSchemaVersion(getSchemaVersionRequest2)).thenReturn(getSchemaVersionResponse2); + } + + private void mockListSchemaVersions(String schemaName, String registryName, Long schemaVersionNumber, Long schemaVersionNumber2) { + ListSchemaVersionsResponse listSchemaVersionsResponse = ListSchemaVersionsResponse.builder() + .schemas(SchemaVersionListItem. + builder(). + schemaArn("test/"+ schemaName). + schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()). + versionNumber(schemaVersionNumber). + status("CREATED"). + build(), + SchemaVersionListItem. + builder(). + schemaArn("test/"+ schemaName). + schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()). + versionNumber(schemaVersionNumber2). + status("CREATED"). + build() + ) + .nextToken(null) + .build(); + ListSchemaVersionsRequest listSchemaVersionsRequest = ListSchemaVersionsRequest.builder() + .schemaId(SchemaId.builder().schemaName(schemaName).registryName(registryName).build()) + .build(); + + when(mockSourceRegistryGlueClient.listSchemaVersions(listSchemaVersionsRequest)).thenReturn(listSchemaVersionsResponse); + } + + private void mockRegisterSchemaVersion(String schemaName, String registryName, Long schemaVersionNumber) { + RegisterSchemaVersionRequest registerSchemaVersionRequest = RegisterSchemaVersionRequest.builder() + .schemaDefinition(userSchemaDefinition) + .schemaId(SchemaId.builder().schemaName(schemaName).registryName(registryName).build()) + .build(); + RegisterSchemaVersionResponse registerSchemaVersionResponse = RegisterSchemaVersionResponse.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .versionNumber(schemaVersionNumber) + .status(SchemaVersionStatus.AVAILABLE) + .build(); + when(mockGlueClient.registerSchemaVersion(registerSchemaVersionRequest)).thenReturn(registerSchemaVersionResponse); + } + + private void mockRegisterSchemaVersion2(String schemaName, String registryName, Long schemaVersionNumber) { + RegisterSchemaVersionRequest registerSchemaVersionRequest = RegisterSchemaVersionRequest.builder() + .schemaDefinition(userSchemaDefinition2) + .schemaId(SchemaId.builder().schemaName(schemaName).registryName(registryName).build()) + .build(); + RegisterSchemaVersionResponse registerSchemaVersionResponse = RegisterSchemaVersionResponse.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) + .versionNumber(schemaVersionNumber) + .status(SchemaVersionStatus.AVAILABLE) + .build(); + when(mockGlueClient.registerSchemaVersion(registerSchemaVersionRequest)).thenReturn(registerSchemaVersionResponse); + } + + private void mockCreateSchema(String schemaName, String dataFormatName, GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration) { + CreateSchemaResponse createSchemaResponse = CreateSchemaResponse.builder() + .schemaName(schemaName) + .dataFormat(dataFormatName) + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .build(); + CreateSchemaRequest createSchemaRequest = CreateSchemaRequest.builder() + .dataFormat(DataFormat.AVRO) + .description(glueSchemaRegistryConfiguration.getDescription()) + .schemaName(schemaName) + .schemaDefinition(userSchemaDefinition) + .compatibility(Compatibility.FORWARD_ALL) + .tags(glueSchemaRegistryConfiguration.getTags()) + .registryId(RegistryId.builder().registryName(glueSchemaRegistryConfiguration.getRegistryName()).build()) + .build(); + + when(mockGlueClient.createSchema(createSchemaRequest)).thenReturn(createSchemaResponse); + } + @Test public void testRegisterSchemaVersion_validParameters_returnsResponseWithSchemaVersionId() throws NoSuchFieldException, IllegalAccessException { awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/SchemaByDefinitionFetcherTestV2.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/SchemaByDefinitionFetcherTestV2.java new file mode 100644 index 00000000..4dc0cf39 --- /dev/null +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/SchemaByDefinitionFetcherTestV2.java @@ -0,0 +1,549 @@ +package com.amazonaws.services.schemaregistry.common; + +public class SchemaByDefinitionFetcherTestV2 { +// private static final UUID SCHEMA_ID_FOR_TESTING = UUID.fromString("f8b4a7f0-9c96-4e4a-a687-fb5de9ef0c63"); +// private static final UUID SCHEMA_ID_FOR_TESTING2 = UUID.fromString("310153e9-9a54-4b12-a513-a23fc543ed2f"); +// private AWSSchemaRegistryClient awsSchemaRegistryClient; +// private SchemaByDefinitionFetcher schemaByDefinitionFetcher; +// +// private GlueClient mockGlueClient; +// private String userSchemaDefinition; +// private String userSchemaDefinition2; +// +// @BeforeEach +// void setUp() { +// mockGlueClient = mock(GlueClient.class); +// awsSchemaRegistryClient = new AWSSchemaRegistryClient(mockGlueClient, mockGlueClient); +// GlueSchemaRegistryConfiguration config = new GlueSchemaRegistryConfiguration(getConfigsWithAutoRegistrationSetting(true)); +// schemaByDefinitionFetcher = new SchemaByDefinitionFetcher(awsSchemaRegistryClient, config); +// userSchemaDefinition = "{Some-avro-schema}"; +// userSchemaDefinition2 = "{Some-avro-schema-v2}"; +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_schemaVersionNotPresent_autoRegistersSchemaVersion() throws Exception { +// Map configs = getConfigsWithAutoRegistrationSetting(true); +// +// String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); +// String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); +// String dataFormatName = DataFormat.AVRO.name(); +// +// Long schemaVersionNumber = 1L; +// SchemaId requestSchemaId = SchemaId.builder().schemaName(schemaName).registryName(registryName).build(); +// +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); +// awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, +// glueSchemaRegistryConfiguration); +// +// mockGetSchemaByDefinition_SchemaVersionNotFoundMsg(schemaName, registryName); +// mockRegisterSchemaVersion(schemaVersionNumber, requestSchemaId); +// mockGetSchemaVersions(schemaVersionNumber, 2L); +// +// schemaByDefinitionFetcher = new SchemaByDefinitionFetcher(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); +// +// UUID schemaVersionId = +// schemaByDefinitionFetcher +// .getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata()); +// +// assertEquals(SCHEMA_ID_FOR_TESTING, schemaVersionId); +// } +// +// private void mockGetSchemaByDefinition_SchemaVersionNotFoundMsg(String schemaName, String registryName) { +// GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = awsSchemaRegistryClient +// .buildGetSchemaByDefinitionRequest(userSchemaDefinition, schemaName, registryName); +// +// EntityNotFoundException entityNotFoundException = +// EntityNotFoundException.builder().message(AWSSchemaRegistryConstants.SCHEMA_VERSION_NOT_FOUND_MSG) +// .build(); +// AWSSchemaRegistryException awsSchemaRegistryException = new AWSSchemaRegistryException(entityNotFoundException); +// +// when(mockGlueClient.getSchemaByDefinition(getSchemaByDefinitionRequest)).thenThrow(awsSchemaRegistryException); +// } +// +// private void mockRegisterSchemaVersion(Long schemaVersionNumber, SchemaId requestSchemaId) { +// RegisterSchemaVersionRequest registerSchemaVersionRequest = RegisterSchemaVersionRequest.builder() +// .schemaDefinition(userSchemaDefinition) +// .schemaId(requestSchemaId) +// .build(); +// RegisterSchemaVersionResponse registerSchemaVersionResponse = RegisterSchemaVersionResponse.builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) +// .versionNumber(schemaVersionNumber) +// .build(); +// when(mockGlueClient.registerSchemaVersion(registerSchemaVersionRequest)) +// .thenReturn(registerSchemaVersionResponse); +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_nullSchemaDefinition_throwsException() { +// Assertions.assertThrows(IllegalArgumentException.class, () -> schemaByDefinitionFetcher +// .getORRegisterSchemaVersionIdV2(null, "test-schema-name", DataFormat.AVRO.name(), Compatibility.FORWARD, getMetadata())); +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_nullSchemaSchemaName_throwsException() { +// Assertions.assertThrows(IllegalArgumentException.class, () -> schemaByDefinitionFetcher +// .getORRegisterSchemaVersionIdV2(userSchemaDefinition, null, DataFormat.AVRO.name(), Compatibility.FORWARD, getMetadata())); +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_nullSchemaDataFormat_throwsException() { +// Assertions.assertThrows(IllegalArgumentException.class, () -> schemaByDefinitionFetcher +// .getORRegisterSchemaVersionIdV2(userSchemaDefinition, "", null, Compatibility.FORWARD, getMetadata())); +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_nullMetadata_throwsException() { +// Assertions.assertThrows(IllegalArgumentException.class, () -> schemaByDefinitionFetcher +// .getORRegisterSchemaVersionIdV2(userSchemaDefinition, "", DataFormat.AVRO.toString(), Compatibility.FORWARD, null)); +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_WhenVersionIsPresent_ReturnsIt() throws Exception { +// Map configs = getConfigsWithAutoRegistrationSetting(true); +// +// String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); +// String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); +// String dataFormatName = DataFormat.AVRO.name(); +// +// GlueSchemaRegistryConfiguration awsSchemaRegistrySerDeConfigs = new GlueSchemaRegistryConfiguration(configs); +// awsSchemaRegistryClient = +// configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, awsSchemaRegistrySerDeConfigs); +// +// mockGetSchemaByDefinition(schemaName, registryName); +// +// schemaByDefinitionFetcher = new SchemaByDefinitionFetcher(awsSchemaRegistryClient, awsSchemaRegistrySerDeConfigs); +// +// UUID schemaVersionId = +// schemaByDefinitionFetcher +// .getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata()); +// +// assertEquals(SCHEMA_ID_FOR_TESTING, schemaVersionId); +// } +// +// private void mockGetSchemaByDefinition(String schemaName, String registryName) { +// GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = awsSchemaRegistryClient +// .buildGetSchemaByDefinitionRequest(userSchemaDefinition, schemaName, registryName); +// +// GetSchemaByDefinitionResponse getSchemaByDefinitionResponse = +// GetSchemaByDefinitionResponse +// .builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) +// .status(SchemaVersionStatus.AVAILABLE) +// .build(); +// +// when(mockGlueClient.getSchemaByDefinition(getSchemaByDefinitionRequest)) +// .thenReturn(getSchemaByDefinitionResponse); +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_OnUnknownException_ThrowsException() throws Exception { +// Map configs = getConfigsWithAutoRegistrationSetting(true); +// +// String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); +// String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); +// String dataFormatName = DataFormat.AVRO.name(); +// +// GlueSchemaRegistryConfiguration awsSchemaRegistrySerDeConfigs = new GlueSchemaRegistryConfiguration(configs); +// awsSchemaRegistryClient = +// configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, awsSchemaRegistrySerDeConfigs); +// +// mockGetSchemaByDefinition_ThrowException(schemaName, registryName); +// +// schemaByDefinitionFetcher = new SchemaByDefinitionFetcher(awsSchemaRegistryClient, awsSchemaRegistrySerDeConfigs); +// +// Exception exception = assertThrows(AWSSchemaRegistryException.class, +// () -> schemaByDefinitionFetcher +// .getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata())); +// assertTrue( +// exception.getMessage().contains("Exception occurred while fetching or registering schema definition")); +// } +// +// private void mockGetSchemaByDefinition_ThrowException(String schemaName, String registryName) { +// GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = awsSchemaRegistryClient +// .buildGetSchemaByDefinitionRequest(userSchemaDefinition, schemaName, registryName); +// +// AWSSchemaRegistryException awsSchemaRegistryException = +// new AWSSchemaRegistryException(new RuntimeException("Unknown")); +// +// when(mockGlueClient.getSchemaByDefinition(getSchemaByDefinitionRequest)).thenThrow(awsSchemaRegistryException); +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_schemaNotPresent_autoCreatesSchema() throws Exception { +// Map configs = getConfigsWithAutoRegistrationSetting(true); +// +// String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); +// String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); +// String dataFormatName = DataFormat.AVRO.name(); +// Long schemaVersionNumber = 1L; +// Long schemaVersionNumber2 = 2L; +// +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); +// awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, +// glueSchemaRegistryConfiguration); +// +// mockGetSchemaByDefinition_SchemaNotFoundMsg(schemaName, registryName); +// mockListSchemaVersions(schemaName, registryName, schemaVersionNumber, schemaVersionNumber2); +// mockCreateSchema(schemaName, dataFormatName, glueSchemaRegistryConfiguration); +// mockGetSchemaVersions(schemaVersionNumber, schemaVersionNumber2); +// mockQuerySchemaVersionMetadata(); +// mockRegisterSchemaVersion2(schemaName, registryName, schemaVersionNumber); +// +// schemaByDefinitionFetcher = new SchemaByDefinitionFetcher(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); +// +// UUID schemaVersionId = schemaByDefinitionFetcher +// .getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata()); +// +// assertEquals(SCHEMA_ID_FOR_TESTING, schemaVersionId); +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_schemaNotPresent_autoCreatesSchemaAndRegisterSchemaVersions_retrieveFromCache() throws Exception { +// Map configs = getConfigsWithAutoRegistrationSetting(true); +// +// String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); +// String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); +// String dataFormatName = DataFormat.AVRO.name(); +// Long schemaVersionNumber = 1L; +// Long schemaVersionNumber2 = 2L; +// +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); +// awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, +// glueSchemaRegistryConfiguration); +// +// GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = mockGetSchemaByDefinition_SchemaNotFoundMsg(schemaName, registryName); +// mockListSchemaVersions(schemaName, registryName, schemaVersionNumber, schemaVersionNumber2); +// mockCreateSchema(schemaName, dataFormatName, glueSchemaRegistryConfiguration); +// mockGetSchemaVersions(schemaVersionNumber, schemaVersionNumber2); +// mockQuerySchemaVersionMetadata(); +// mockRegisterSchemaVersion2(schemaName, registryName, schemaVersionNumber); +// +// schemaByDefinitionFetcher = new SchemaByDefinitionFetcher(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); +// +// LoadingCache cacheV2 = schemaByDefinitionFetcher.schemaDefinitionToVersionCacheV2; +// +// SchemaV2 expectedSchema = new SchemaV2(userSchemaDefinition, dataFormatName, schemaName, Compatibility.FORWARD); +// SchemaV2 expectedSchema2 = new SchemaV2(userSchemaDefinition2, dataFormatName, schemaName, Compatibility.FORWARD); +// +// //Ensure cache is empty to start with. +// assertEquals(0, cacheV2.size()); +// +// //First call will create schema and register other schema versions +// schemaByDefinitionFetcher.getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata()); +// +// //Ensure cache is populated +// assertEquals(2, cacheV2.size()); +// +// //Ensure corresponding UUID matches with schema +// assertEquals(SCHEMA_ID_FOR_TESTING, cacheV2.get(expectedSchema)); +// assertEquals(SCHEMA_ID_FOR_TESTING2, cacheV2.get(expectedSchema2)); +// +// //Second call will be served from cache +// schemaByDefinitionFetcher.getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata()); +// +// //Third call will be served from cache +// schemaByDefinitionFetcher.getORRegisterSchemaVersionIdV2(userSchemaDefinition2, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata()); +// +// //Ensure cache is populated +// assertEquals(2, cacheV2.size()); +// +// //Ensure only 1 call happened. +// verify(mockGlueClient, times(1)).getSchemaByDefinition(getSchemaByDefinitionRequest); +// } +// +// private void mockRegisterSchemaVersion2(String schemaName, String registryName, Long schemaVersionNumber) { +// RegisterSchemaVersionRequest registerSchemaVersionRequest = RegisterSchemaVersionRequest.builder() +// .schemaDefinition(userSchemaDefinition2) +// .schemaId(SchemaId.builder().schemaName(schemaName).registryName(registryName).build()) +// .build(); +// RegisterSchemaVersionResponse registerSchemaVersionResponse = RegisterSchemaVersionResponse.builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) +// .versionNumber(schemaVersionNumber) +// .status(SchemaVersionStatus.AVAILABLE) +// .build(); +// when(mockGlueClient.registerSchemaVersion(registerSchemaVersionRequest)).thenReturn(registerSchemaVersionResponse); +// } +// +// private GetSchemaByDefinitionRequest mockGetSchemaByDefinition_SchemaNotFoundMsg(String schemaName, String registryName) { +// GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = awsSchemaRegistryClient +// .buildGetSchemaByDefinitionRequest(userSchemaDefinition, schemaName, registryName); +// +// EntityNotFoundException entityNotFoundException = +// EntityNotFoundException.builder().message(AWSSchemaRegistryConstants.SCHEMA_NOT_FOUND_MSG) +// .build(); +// AWSSchemaRegistryException awsSchemaRegistryException = new AWSSchemaRegistryException(entityNotFoundException); +// +// when(mockGlueClient.getSchemaByDefinition(getSchemaByDefinitionRequest)).thenThrow(awsSchemaRegistryException); +// +// return getSchemaByDefinitionRequest; +// } +// +// private void mockQuerySchemaVersionMetadata() { +// QuerySchemaVersionMetadataRequest querySchemaVersionMetadataRequest = QuerySchemaVersionMetadataRequest.builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) +// .build(); +// +// QuerySchemaVersionMetadataResponse querySchemaVersionMetadataResponse = QuerySchemaVersionMetadataResponse +// .builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) +// .metadataInfoMap(new HashMap<>()) +// .build(); +// +// QuerySchemaVersionMetadataRequest querySchemaVersionMetadataRequest2 = QuerySchemaVersionMetadataRequest.builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) +// .build(); +// +// QuerySchemaVersionMetadataResponse querySchemaVersionMetadataResponse2 = QuerySchemaVersionMetadataResponse +// .builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) +// .metadataInfoMap(new HashMap<>()) +// .build(); +// +// when(mockGlueClient.querySchemaVersionMetadata(querySchemaVersionMetadataRequest)).thenReturn(querySchemaVersionMetadataResponse); +// when(mockGlueClient.querySchemaVersionMetadata(querySchemaVersionMetadataRequest2)).thenReturn(querySchemaVersionMetadataResponse2); +// } +// +// private void mockGetSchemaVersions(Long schemaVersionNumber, Long schemaVersionNumber2) { +// GetSchemaVersionRequest getSchemaVersionRequest = GetSchemaVersionRequest.builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()).build(); +// +// GetSchemaVersionResponse getSchemaVersionResponse = GetSchemaVersionResponse.builder() +// .schemaDefinition(userSchemaDefinition) +// .versionNumber(schemaVersionNumber) +// .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) +// .dataFormat(DataFormat.AVRO) +// .status(SchemaVersionStatus.AVAILABLE) +// .build(); +// +// when(mockGlueClient.getSchemaVersion(getSchemaVersionRequest)).thenReturn(getSchemaVersionResponse); +// +// GetSchemaVersionRequest getSchemaVersionRequest2 = GetSchemaVersionRequest.builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()).build(); +// +// GetSchemaVersionResponse getSchemaVersionResponse2 = GetSchemaVersionResponse.builder() +// .schemaDefinition(userSchemaDefinition2) +// .versionNumber(schemaVersionNumber2) +// .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) +// .dataFormat(DataFormat.AVRO) +// .status(SchemaVersionStatus.AVAILABLE) +// .build(); +// +// when(mockGlueClient.getSchemaVersion(getSchemaVersionRequest2)).thenReturn(getSchemaVersionResponse2); +// } +// +// private void mockCreateSchema(String schemaName, String dataFormatName, GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration) { +// CreateSchemaResponse createSchemaResponse = CreateSchemaResponse.builder() +// .schemaName(schemaName) +// .dataFormat(dataFormatName) +// .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) +// .build(); +// CreateSchemaRequest createSchemaRequest = CreateSchemaRequest.builder() +// .dataFormat(DataFormat.AVRO) +// .description(glueSchemaRegistryConfiguration.getDescription()) +// .schemaName(schemaName) +// .schemaDefinition(userSchemaDefinition) +// .compatibility(Compatibility.FORWARD) +// .tags(glueSchemaRegistryConfiguration.getTags()) +// .registryId(RegistryId.builder().registryName(glueSchemaRegistryConfiguration.getRegistryName()).build()) +// .build(); +// +// when(mockGlueClient.createSchema(createSchemaRequest)).thenReturn(createSchemaResponse); +// } +// +// private void mockListSchemaVersions(String schemaName, String registryName, Long schemaVersionNumber, Long schemaVersionNumber2) { +// ListSchemaVersionsResponse listSchemaVersionsResponse = ListSchemaVersionsResponse.builder() +// .schemas(SchemaVersionListItem. +// builder(). +// schemaArn("test/"+ schemaName). +// schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()). +// versionNumber(schemaVersionNumber). +// status("CREATED"). +// build(), +// SchemaVersionListItem. +// builder(). +// schemaArn("test/"+ schemaName). +// schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()). +// versionNumber(schemaVersionNumber2). +// status("CREATED"). +// build() +// ) +// .nextToken(null) +// .build(); +// ListSchemaVersionsRequest listSchemaVersionsRequest = ListSchemaVersionsRequest.builder() +// .schemaId(SchemaId.builder().schemaName(schemaName).registryName(registryName).build()) +// .build(); +// +// when(mockGlueClient.listSchemaVersions(listSchemaVersionsRequest)).thenReturn(listSchemaVersionsResponse); +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_autoRegistrationDisabled_failsIfSchemaVersionNotPresent() +// throws Exception { +// Map configs = getConfigsWithAutoRegistrationSetting(false); +// +// String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); +// String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); +// String dataFormatName = DataFormat.AVRO.name(); +// +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); +// awsSchemaRegistryClient = +// configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, +// glueSchemaRegistryConfiguration); +// +// mockGetSchemaByDefinition_SchemaNotFoundMsg(schemaName, registryName); +// +// schemaByDefinitionFetcher = new SchemaByDefinitionFetcher(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); +// +// Exception exception = assertThrows(AWSSchemaRegistryException.class, () -> schemaByDefinitionFetcher +// .getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata())); +// +// assertEquals(AWSSchemaRegistryConstants.AUTO_REGISTRATION_IS_DISABLED_MSG, exception.getMessage()); +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_retrieveSchemaVersionId_schemaVersionIdIsCached() throws Exception { +// Map configs = getConfigsWithAutoRegistrationSetting(false); +// +// String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); +// String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); +// String dataFormatName = DataFormat.AVRO.name(); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); +// +// GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = mockGetSchemaByDefinition_RetrieveFromCache(schemaName, registryName); +// +// awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); +// schemaByDefinitionFetcher = new SchemaByDefinitionFetcher(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); +// LoadingCache cacheV2 = schemaByDefinitionFetcher.schemaDefinitionToVersionCacheV2; +// +// //Ensure cache is empty to start with. +// assertEquals(0, cacheV2.size()); +// +// //First call +// schemaByDefinitionFetcher.getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata()); +// //Second call +// schemaByDefinitionFetcher.getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata()); +// //Third call +// schemaByDefinitionFetcher.getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata()); +// +// //Ensure cache is populated +// assertEquals(1, cacheV2.size()); +// +// SchemaV2 expectedSchema = new SchemaV2(userSchemaDefinition, dataFormatName, schemaName, Compatibility.FORWARD); +// Map.Entry cacheEntry = (Map.Entry) cacheV2.asMap().entrySet().toArray()[0]; +// +// //Ensure cache entries are expected +// assertEquals(expectedSchema, cacheEntry.getKey()); +// assertEquals(SCHEMA_ID_FOR_TESTING, cacheEntry.getValue()); +// +// //Ensure only 1 call happened. +// verify(mockGlueClient, times(1)).getSchemaByDefinition(getSchemaByDefinitionRequest); +// } +// +// private GetSchemaByDefinitionRequest mockGetSchemaByDefinition_RetrieveFromCache(String schemaName, String registryName) { +// GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = awsSchemaRegistryClient +// .buildGetSchemaByDefinitionRequest(userSchemaDefinition, schemaName, registryName); +// +// GetSchemaByDefinitionResponse getSchemaByDefinitionResponse = +// GetSchemaByDefinitionResponse +// .builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) +// .status(SchemaVersionStatus.AVAILABLE) +// .build(); +// +// when(mockGlueClient.getSchemaByDefinition(getSchemaByDefinitionRequest)) +// .thenReturn(getSchemaByDefinitionResponse); +// return getSchemaByDefinitionRequest; +// } +// +// @Test +// public void testGetORRegisterSchemaVersionIdV2_continuesToServeFromCache_WhenCallsFail() throws Exception { +// Map configs = getConfigsWithAutoRegistrationSetting(false); +// +// String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); +// String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); +// String dataFormatName = DataFormat.AVRO.name(); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); +// +// //Override TTL to 1s. +// glueSchemaRegistryConfiguration.setTimeToLiveMillis(1000L); +// +// GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = awsSchemaRegistryClient +// .buildGetSchemaByDefinitionRequest(userSchemaDefinition, schemaName, registryName); +// +// GetSchemaByDefinitionResponse getSchemaByDefinitionResponse = +// GetSchemaByDefinitionResponse +// .builder() +// .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) +// .status(software.amazon.awssdk.services.glue.model.SchemaVersionStatus.AVAILABLE) +// .build(); +// +// awsSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); +// schemaByDefinitionFetcher = new SchemaByDefinitionFetcher(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); +// LoadingCache cacheV2 = schemaByDefinitionFetcher.schemaDefinitionToVersionCacheV2; +// +// //Ensure cache is empty to start with. +// assertEquals(0, cacheV2.size()); +// +// //Mock the client to return response, then fail and eventually succeed. +// when(mockGlueClient.getSchemaByDefinition(getSchemaByDefinitionRequest)) +// .thenReturn(getSchemaByDefinitionResponse) +// .thenThrow(new RuntimeException("Service outage")) +// .thenThrow(new RuntimeException("Service outage")) +// .thenReturn(getSchemaByDefinitionResponse); +// +// //First call +// //As expected first call should fetch and cache the schema version. +// assertDoesNotThrow(() -> schemaByDefinitionFetcher.getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata())); +// assertEquals(1, cacheV2.size()); +// +// //Wait for 1.5 seconds to expire cacheV2. +// Thread.sleep(1500L); +// +// //Second call shouldn't fail. +// assertDoesNotThrow(() -> schemaByDefinitionFetcher.getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata())); +// +// //Third call shouldn't fail. +// assertDoesNotThrow(() -> schemaByDefinitionFetcher.getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata())); +// +// //Verify the entry is not evicted. +// assertEquals(1, cacheV2.size()); +// +// //Fourth call shouldn't fail and cache is refreshed. +// assertDoesNotThrow(() -> schemaByDefinitionFetcher.getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata())); +// verify(mockGlueClient, times(4)).getSchemaByDefinition(getSchemaByDefinitionRequest); +// } +// +// private Map getConfigsWithAutoRegistrationSetting(boolean autoRegistrationSetting) { +// Map localConfigs = new HashMap<>(); +// localConfigs.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, "https://test"); +// localConfigs.put(AWSSchemaRegistryConstants.AWS_REGION, "us-west-2"); +// localConfigs.put(AWSSchemaRegistryConstants.SCHEMA_NAME, "User-Topic"); +// localConfigs.put(AWSSchemaRegistryConstants.REGISTRY_NAME, "User-Topic"); +// localConfigs.put(AWSSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "User-Topic"); +// localConfigs.put(AWSSchemaRegistryConstants.SCHEMA_AUTO_REGISTRATION_SETTING, +// String.valueOf(autoRegistrationSetting)); +// return localConfigs; +// +// } +// +// private AWSSchemaRegistryClient configureAWSSchemaRegistryClientWithSerdeConfig( +// AWSSchemaRegistryClient awsSchemaRegistryClient, +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration) +// throws NoSuchFieldException, IllegalAccessException { +// Field serdeConfigField = AWSSchemaRegistryClient.class.getDeclaredField("glueSchemaRegistryConfiguration"); +// serdeConfigField.setAccessible(true); +// serdeConfigField.set(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); +// +// return awsSchemaRegistryClient; +// } +// +// private Map getMetadata() { +// Map metadata = new HashMap<>(); +// metadata.put("event-source-1", "topic1"); +// metadata.put("event-source-2", "topic2"); +// metadata.put("event-source-3", "topic3"); +// metadata.put("event-source-4", "topic4"); +// metadata.put("event-source-5", "topic5"); +// return metadata; +// } +} \ No newline at end of file diff --git a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java index 70613e06..2e9c40a9 100644 --- a/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java +++ b/common/src/test/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfigurationTest.java @@ -388,6 +388,123 @@ public void testValidateAndSetRegistryName_withRegistryConfig_throwsException() assertEquals(expectedRegistryName, glueSchemaRegistryConfiguration.getRegistryName()); } + /** + * Tests source registry name + */ +// @Test +// public void testValidateAndSetSourceRegistryName_withSourceRegistryConfig() { +// String expectedRegistryName = "test-registry"; +// Properties props = createTestProperties(); +// props.put(AWSSchemaRegistryConstants.SOURCE_REGISTRY_NAME, expectedRegistryName); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); +// +// assertEquals(expectedRegistryName, glueSchemaRegistryConfiguration.getSourceRegistryName()); +// } + + /** + * Tests target registry name + */ +// @Test +// public void testValidateAndSetTargetRegistryName_withTargetRegistryConfig() { +// String expectedRegistryName = "test-registry"; +// Properties props = createTestProperties(); +// props.put(AWSSchemaRegistryConstants.TARGET_REGISTRY_NAME, expectedRegistryName); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); +// +// assertEquals(expectedRegistryName, glueSchemaRegistryConfiguration.getTargetRegistryName()); +// } + + /** + * Tests target registry name + */ +// @Test +// public void testValidateAndSetTargetRegistryName_withRegistryConfig() { +// String expectedRegistryName = "test-registry"; +// Properties props = createTestProperties(); +// props.put(AWSSchemaRegistryConstants.REGISTRY_NAME, expectedRegistryName); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); +// +// assertEquals(expectedRegistryName, glueSchemaRegistryConfiguration.getTargetRegistryName()); +// } + + /** + * Tests target endpoint + */ +// @Test +// public void testValidateAndSetTargetEndpoint_withTargetEndpointConfig() { +// String expectedEndpoint = "http://glue.us-east-1"; +// Properties props = createTestProperties(); +// props.put(AWSSchemaRegistryConstants.AWS_TARGET_ENDPOINT, expectedEndpoint); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); +// +// assertEquals(expectedEndpoint, glueSchemaRegistryConfiguration.getTargetEndPoint()); +// } + + /** + * Tests target endpoint + */ +// @Test +// public void testValidateAndSetTargetEndpoint_withEndpointConfig() { +// String expectedEndpoint = "http://glue.us-east-1"; +// Properties props = createTestProperties(); +// props.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, expectedEndpoint); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); +// +// assertEquals(expectedEndpoint, glueSchemaRegistryConfiguration.getTargetEndPoint()); +// } + + /** + * Tests source endpoint + */ +// @Test +// public void testValidateAndSetSourceEndpoint_withSourceEndpointConfig() { +// String expectedEndpoint = "http://glue.us-east-1"; +// Properties props = createTestProperties(); +// props.put(AWSSchemaRegistryConstants.AWS_SOURCE_ENDPOINT, expectedEndpoint); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); +// +// assertEquals(expectedEndpoint, glueSchemaRegistryConfiguration.getSourceEndPoint()); +// } + + /** + * Tests source region + */ +// @Test +// public void testValidateAndSetSourceRegion_withSourceRegionConfig() { +// String expectedRegion = "us-east-1"; +// Properties props = createTestProperties(); +// props.put(AWSSchemaRegistryConstants.AWS_SOURCE_REGION, expectedRegion); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); +// +// assertEquals(expectedRegion, glueSchemaRegistryConfiguration.getSourceRegion()); +// } + + /** + * Tests target endpoint + */ +// @Test +// public void testValidateAndSetTargetRegion_withTargetRegionConfig() { +// String expectedRegion = "us-east-1"; +// Properties props = createTestProperties(); +// props.put(AWSSchemaRegistryConstants.AWS_TARGET_REGION, expectedRegion); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); +// +// assertEquals(expectedRegion, glueSchemaRegistryConfiguration.getTargetRegion()); +// } + + /** + * Tests target endpoint + */ +// @Test +// public void testValidateAndSetTargetRegion_withRegionConfig() { +// String expectedRegion = "us-east-1"; +// Properties props = createTestProperties(); +// props.put(AWSSchemaRegistryConstants.AWS_REGION, expectedRegion); +// GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(props); +// +// assertEquals(expectedRegion, glueSchemaRegistryConfiguration.getTargetRegion()); +// } + /** * Tests valid proxy URL value. */ diff --git a/cross-region-replication-converter/.gitignore b/cross-region-replication-converter/.gitignore new file mode 100644 index 00000000..8a498597 --- /dev/null +++ b/cross-region-replication-converter/.gitignore @@ -0,0 +1,40 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +resources/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/cross-region-replication-converter/pom.xml b/cross-region-replication-converter/pom.xml new file mode 100644 index 00000000..4ea2e987 --- /dev/null +++ b/cross-region-replication-converter/pom.xml @@ -0,0 +1,221 @@ + + + 4.0.0 + + ${parent.groupId} + schema-registry-cross-region-kafkaconnect-converter + ${parent.version} + AWS Cross Region Glue Schema Registry Kafka Connect Schema Replication Converter + The AWS Glue Schema Registry Kafka Connect Converter enables Java developers to easily replicate + schemas across different AWS Glue Schema Registries + + https://aws.amazon.com/glue + jar + + + software.amazon.glue + schema-registry-parent + 1.1.20 + ../pom.xml + + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + ossrh + https://aws.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://aws.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + scm:git:https://github.com/aws/aws-glue-schema-registry.git + scm:git:git@github.com:aws/aws-glue-schema-registry.git + https://github.com/awslabs/aws-glue-schema-registry.git + + + + + ${parent.groupId} + schema-registry-serde + ${parent.version} + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + maven-plugin + + + + org.apache.kafka + connect-api + + + org.projectlombok + lombok + + + org.mockito + mockito-core + test + + + org.mockito + mockito-junit-jupiter + test + + + org.junit.vintage + junit-vintage-engine + 5.7.0 + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.platform + junit-platform-commons + test + + + org.junit.jupiter + junit-jupiter-api + test + + + junit + junit + test + + + org.powermock + powermock-reflect + 2.0.7 + test + + + uk.co.jemos.podam + podam + 7.2.5.RELEASE + test + + + software.amazon.glue + schema-registry-kafkaconnect-converter + 1.1.20 + compile + + + + + + + maven-shade-plugin + 3.2.1 + + + + + package + + shade + + + + + + org.apache.avro + avro-maven-plugin + ${avro.version} + + + generate-test-sources + test-sources + + schema + + + + generate-sources + sources + + schema + + + + + true + String + + + + + + + publishing + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.8 + true + + sonatype-nexus-staging + https://aws.oss.sonatype.org + false + + + + + + + \ No newline at end of file diff --git a/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverter.java b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverter.java new file mode 100644 index 00000000..34d1b038 --- /dev/null +++ b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverter.java @@ -0,0 +1,324 @@ +package com.amazonaws.services.crossregion.schemaregistry.kafkaconnect; + +import com.amazonaws.services.schemaregistry.common.AWSSchemaRegistryClient; +import com.amazonaws.services.schemaregistry.common.Schema; +import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryDeserializerImpl; +import com.amazonaws.services.schemaregistry.exception.AWSSchemaRegistryException; +import com.amazonaws.services.schemaregistry.exception.GlueSchemaRegistryIncompatibleDataException; +import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistrySerializerImpl; +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import lombok.Data; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.connect.data.SchemaAndValue; +import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.storage.Converter; +import org.jetbrains.annotations.NotNull; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.services.glue.model.AlreadyExistsException; +import software.amazon.awssdk.services.glue.model.Compatibility; +import software.amazon.awssdk.services.glue.model.GetSchemaResponse; +import software.amazon.awssdk.services.glue.model.GetSchemaVersionResponse; +import software.amazon.awssdk.services.glue.model.MetadataInfo; +import software.amazon.awssdk.services.glue.model.QuerySchemaVersionMetadataResponse; +import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.glue.model.SchemaVersionListItem; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + + +@Data +@Slf4j +public class AWSGlueCrossRegionSchemaReplicationConverter implements Converter { + + private AwsCredentialsProvider credentialsProvider; + private GlueSchemaRegistryDeserializerImpl deserializer; + private GlueSchemaRegistrySerializerImpl serializer; + private boolean isKey; + private Map sourceConfigs; + private Map targetConfigs; + private SchemaReplicationGlueSchemaRegistryConfiguration targetGlueSchemaRegistryConfiguration; + private SchemaReplicationGlueSchemaRegistryConfiguration sourceGlueSchemaRegistryConfiguration; + + @NonNull + private AWSSchemaRegistryClient targetSchemaRegistryClient; + @NonNull + private AWSSchemaRegistryClient sourceSchemaRegistryClient; + + @NonNull + @VisibleForTesting + protected LoadingCache schemaDefinitionToVersionCache; + + + /** + * Constructor used by Kafka Connect user. + */ + public AWSGlueCrossRegionSchemaReplicationConverter(){} + + /** + * Constructor accepting AWSCredentialsProvider. + * + * @param credentialsProvider AWSCredentialsProvider instance. + */ + public AWSGlueCrossRegionSchemaReplicationConverter( + AwsCredentialsProvider credentialsProvider, + GlueSchemaRegistryDeserializerImpl deserializerImpl, + GlueSchemaRegistrySerializerImpl serializerImpl) { + + this.credentialsProvider = credentialsProvider; + this.deserializer = deserializerImpl; + this.serializer = serializerImpl; + } + + /** + * Configure the Schema Replication Converter. + * @param configs configuration elements for the converter + * @param isKey true if key, false otherwise + */ + @Override + public void configure(Map configs, boolean isKey) { + this.isKey = isKey; + // TODO: Support credentialProvider passed on by the user + // https://github.com/awslabs/aws-glue-schema-registry/issues/293 + credentialsProvider = DefaultCredentialsProvider.builder().build(); + + // Put the source and target regions into configurations respectively + sourceConfigs = new HashMap<>(configs); + targetConfigs = new HashMap<>(configs); + + validateRequiredConfigsIfPresent(configs); + + if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION) != null) { + sourceConfigs.put(AWSSchemaRegistryConstants.AWS_REGION, configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION)); + } + if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT) != null) { + sourceConfigs.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT)); + } + if (configs.get(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME) != null) { + sourceConfigs.put(AWSSchemaRegistryConstants.REGISTRY_NAME, configs.get(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME)); + } + + if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION) != null) { + targetConfigs.put(AWSSchemaRegistryConstants.AWS_REGION, configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION)); + } + if (configs.get(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME) != null) { + targetConfigs.put(AWSSchemaRegistryConstants.REGISTRY_NAME, configs.get(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME)); + } + if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT) != null) { + targetConfigs.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT)); + } + + targetConfigs.put(AWSSchemaRegistryConstants.SCHEMA_AUTO_REGISTRATION_SETTING, true); + + targetGlueSchemaRegistryConfiguration = new SchemaReplicationGlueSchemaRegistryConfiguration(targetConfigs); + sourceGlueSchemaRegistryConfiguration = new SchemaReplicationGlueSchemaRegistryConfiguration(sourceConfigs); + + targetSchemaRegistryClient = new AWSSchemaRegistryClient(credentialsProvider, targetGlueSchemaRegistryConfiguration); + sourceSchemaRegistryClient = new AWSSchemaRegistryClient(credentialsProvider, sourceGlueSchemaRegistryConfiguration); + + this.schemaDefinitionToVersionCache = CacheBuilder.newBuilder() + .maximumSize(targetGlueSchemaRegistryConfiguration.getCacheSize()) + .refreshAfterWrite(targetGlueSchemaRegistryConfiguration.getTimeToLiveMillis(), TimeUnit.MILLISECONDS) + .build(new SchemaDefinitionToVersionCache()); + + serializer = new GlueSchemaRegistrySerializerImpl(credentialsProvider, targetGlueSchemaRegistryConfiguration); + deserializer = new GlueSchemaRegistryDeserializerImpl(credentialsProvider, sourceGlueSchemaRegistryConfiguration); + } + + @Override + public byte[] fromConnectData(String topic, org.apache.kafka.connect.data.Schema schema, Object value) { + if (value == null) return null; + byte[] bytes = (byte[]) value; + + try { + byte[] deserializedBytes = deserializer.getData(bytes); + Schema deserializedSchema = deserializer.getSchema(bytes); + createSchemaAndRegisterAllSchemaVersions(deserializedSchema); + return serializer.encode(topic, deserializedSchema, deserializedBytes); + } catch(GlueSchemaRegistryIncompatibleDataException ex) { + //This exception is raised when the header bytes don't have schema id, version byte or compression byte + //This determines the data doesn't have schema information in it, so the actual message is returned. + return bytes; + } + catch (SerializationException | AWSSchemaRegistryException e) { + throw new DataException("Converting Kafka Connect data to byte[] failed due to serialization/deserialization error: ", e); + } catch (ExecutionException e) { + //TODO: Proper messaging and error handling + throw new DataException("Converting Kafka Connect data to byte[] failed due to serialization/deserialization error: ", e); + } + } + + private void validateRequiredConfigsIfPresent(Map configs) { + if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION) == null) { + throw new DataException("Source Region is not provided."); + } else if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION) == null && configs.get(AWSSchemaRegistryConstants.AWS_REGION) == null) { + throw new DataException("Target Region is not provided."); + } else if (configs.get(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME) == null) { + throw new DataException("Source Registry is not provided."); + } else if (configs.get(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME) == null && configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME) == null) { + throw new DataException("Target Registry is not provided."); + } else if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT) == null) { + throw new DataException("Source Endpoint is not provided."); + } else if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT) == null && configs.get(AWSSchemaRegistryConstants.AWS_ENDPOINT) == null) { + throw new DataException("Target Endpoint is not provided."); + } + } + + /** + * This method is not intended to be used for the CrossRegionReplicationConverter given it is integrated with a source connector + * + */ + @Override + public SchemaAndValue toConnectData(String topic, byte[] value) { + throw new UnsupportedOperationException("This method is not supported"); + } + + @VisibleForTesting + private UUID createSchemaAndRegisterAllSchemaVersions( + @NonNull Schema schema) throws AWSSchemaRegistryException, ExecutionException { + + UUID schemaVersionId; + + try { + return schemaDefinitionToVersionCache.get(schema); + } catch(Exception ex) { + Map schemaWithVersionId = new HashMap<>(); + String schemaName = schema.getSchemaName(); + String schemaNameFromArn = ""; + String schemaDefinition = ""; + String dataFormat = schema.getDataFormat(); + Map metadataInfo = new HashMap<>(); + GetSchemaVersionResponse schemaVersionResponse = null; + + + //Get compatibility mode for each schema + Compatibility compatibility = getCompatibilityMode(schema); + + targetGlueSchemaRegistryConfiguration.setCompatibilitySetting(compatibility); + targetSchemaRegistryClient = new AWSSchemaRegistryClient(credentialsProvider, targetGlueSchemaRegistryConfiguration); + + try{ + //Get list of all schema versions + List schemaVersionList = sourceSchemaRegistryClient.getSchemaVersions(schemaName, targetGlueSchemaRegistryConfiguration.getReplicateSchemaVersionCount()); + + for (int idx = 0; idx < schemaVersionList.size(); idx++){ + //Get details of each schema versions + schemaVersionResponse = + sourceSchemaRegistryClient.getSchemaVersionResponse(schemaVersionList.get(idx).schemaVersionId()); + + schemaNameFromArn = getSchemaNameFromArn(schemaVersionList.get(idx).schemaArn()); + schemaDefinition = schemaVersionResponse.schemaDefinition(); + + //Get the metadata information for each version + QuerySchemaVersionMetadataResponse querySchemaVersionMetadataResponse = sourceSchemaRegistryClient.querySchemaVersionMetadata(UUID.fromString(schemaVersionResponse.schemaVersionId())); + metadataInfo = getMetadataInfo(querySchemaVersionMetadataResponse.metadataInfoMap()); + //Create the schema with the first schema version + if (idx == 0) { + //Create the schema + schemaVersionId = createSchema(schemaNameFromArn, schemaDefinition, dataFormat, metadataInfo, schemaVersionResponse); + } else { + //Register subsequent schema versions + schemaVersionId = targetSchemaRegistryClient.registerSchemaVersion(schemaVersionResponse.schemaDefinition(), + schemaNameFromArn, dataFormat, metadataInfo); + } + + cacheAllSchemaVersions(schemaVersionId, schemaWithVersionId, schemaNameFromArn, schemaVersionResponse); + } + + } + catch (AlreadyExistsException e) { + log.warn("Schema is already created, this could be caused by multiple producers/MM2 racing to auto-create schema."); + schemaVersionId = targetSchemaRegistryClient.registerSchemaVersion(schemaDefinition, schemaName, dataFormat, metadataInfo); + cacheAllSchemaVersions(schemaVersionId, schemaWithVersionId, schemaNameFromArn, schemaVersionResponse); + targetSchemaRegistryClient.putSchemaVersionMetadata(schemaVersionId, metadataInfo); + } + catch (Exception e) { + String errorMessage = String.format( + "Create schema :: Call failed when creating the schema with the schema registry for" + + " schema name = %s", schemaName); + //TODO: Will this exception be ever thrown? + throw new AWSSchemaRegistryException(errorMessage, e); + } + } + + schemaVersionId = schemaDefinitionToVersionCache.get(schema); + return schemaVersionId; + } + + private void cacheAllSchemaVersions(UUID schemaVersionId, Map schemaWithVersionId, String schemaNameFromArn, GetSchemaVersionResponse getSchemaVersionResponse) { + Schema schemaVersionSchema = new Schema(getSchemaVersionResponse.schemaDefinition(), getSchemaVersionResponse.dataFormat().toString(), schemaNameFromArn); + + //Create a map of schema and schemaVersionId + schemaWithVersionId.put(schemaVersionSchema, schemaVersionId); + //Cache all the schema versions for a Glue Schema Registry schema + schemaWithVersionId.entrySet() + .stream() + .forEach(item -> { + schemaDefinitionToVersionCache.put(item.getKey(), item.getValue()); + }); + } + + private UUID createSchema(String schemaNameFromArn, String schemaDefinition, String dataFormat, Map metadataInfo, GetSchemaVersionResponse getSchemaVersionResponse) { + UUID schemaVersionId; + log.info("Auto Creating schema with schemaName: {} and schemaDefinition : {}", + schemaNameFromArn, getSchemaVersionResponse.schemaDefinition()); + + schemaVersionId = targetSchemaRegistryClient.createSchema( + schemaNameFromArn, + dataFormat, + schemaDefinition, new HashMap<>()); //TODO: Get metadata of Schema + + //Add version metadata to the schema version + targetSchemaRegistryClient.putSchemaVersionMetadata(schemaVersionId, metadataInfo); + return schemaVersionId; + } + + private Compatibility getCompatibilityMode(@NotNull Schema schema) { + GetSchemaResponse schemaResponse = sourceSchemaRegistryClient.getSchemaResponse(SchemaId.builder() + .schemaName(schema.getSchemaName()) + .registryName(sourceGlueSchemaRegistryConfiguration.getSourceRegistryName()) + .build()); + + Compatibility compatibility = schemaResponse.compatibility(); + return compatibility; + } + + private String getSchemaNameFromArn(String schemaArn) { + String[] tokens = schemaArn.split(Pattern.quote("/")); + return tokens[tokens.length - 1]; + } + + private Map getMetadataInfo(Map metadataInfoMap) { + Map metadata = new HashMap<>(); + Iterator> iterator = metadataInfoMap.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + metadata.put(entry.getKey(), entry.getValue().metadataValue()); + } + + return metadata; + } + + @RequiredArgsConstructor + private class SchemaDefinitionToVersionCache extends CacheLoader { + @Override + public UUID load(Schema schema) { + return targetSchemaRegistryClient.getSchemaVersionIdByDefinition( + schema.getSchemaDefinition(), schema.getSchemaName(), schema.getDataFormat()); + } + } +} diff --git a/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationGlueSchemaRegistryConfiguration.java b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationGlueSchemaRegistryConfiguration.java new file mode 100644 index 00000000..d11242eb --- /dev/null +++ b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationGlueSchemaRegistryConfiguration.java @@ -0,0 +1,84 @@ +package com.amazonaws.services.crossregion.schemaregistry.kafkaconnect; + +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; + +@Slf4j +@Getter +public class SchemaReplicationGlueSchemaRegistryConfiguration extends GlueSchemaRegistryConfiguration { + private String sourceEndPoint; + private String sourceRegion; + private String targetEndPoint; + private String targetRegion; + private String sourceRegistryName; + private String targetRegistryName; + private int replicateSchemaVersionCount; + + public SchemaReplicationGlueSchemaRegistryConfiguration(Map configs) { + super(configs); + buildSchemaReplicationSchemaRegistryConfigs(configs); + } + + private void buildSchemaReplicationSchemaRegistryConfigs(Map configs) { + validateAndSetAWSSourceRegion(configs); + validateAndSetAWSTargetRegion(configs); + validateAndSetAWSSourceEndpoint(configs); + validateAndSetAWSTargetEndpoint(configs); + validateAndSetSourceRegistryName(configs); + validateAndSetTargetRegistryName(configs); + validateAndSetReplicateSchemaVersionCount(configs); + } + + private void validateAndSetAWSSourceRegion(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION)) { + this.sourceRegion = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION)); + } + } + + private void validateAndSetAWSTargetRegion(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION)) { + this.targetRegion = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION)); + } else { + this.targetRegion = this.getRegion(); + } + } + + private void validateAndSetSourceRegistryName(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME)) { + this.sourceRegistryName = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME)); + } + } + + private void validateAndSetTargetRegistryName(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME)) { + this.targetRegistryName = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME)); + } else { + this.targetRegistryName = this.getRegistryName(); + } + } + + private void validateAndSetAWSSourceEndpoint(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT)) { + this.sourceEndPoint = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT)); + } + } + + private void validateAndSetAWSTargetEndpoint(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT)) { + this.targetEndPoint = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT)); + } else { + this.targetEndPoint = this.getEndPoint(); + } + } + + private void validateAndSetReplicateSchemaVersionCount(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.REPLICATE_SCHEMA_VERSION_COUNT)) { + this.replicateSchemaVersionCount = Integer.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.REPLICATE_SCHEMA_VERSION_COUNT).toString()); + } else { + this.replicateSchemaVersionCount = SchemaReplicationSchemaRegistryConstants.DEFAULT_REPLICATE_SCHEMA_VERSION_COUNT; + } + } +} diff --git a/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationSchemaRegistryConstants.java b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationSchemaRegistryConstants.java new file mode 100644 index 00000000..3f3d3089 --- /dev/null +++ b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationSchemaRegistryConstants.java @@ -0,0 +1,36 @@ +package com.amazonaws.services.crossregion.schemaregistry.kafkaconnect; + +public class SchemaReplicationSchemaRegistryConstants { + /** + * AWS source endpoint to use while initializing the client for service. + */ + public static final String AWS_SOURCE_ENDPOINT = "source.endpoint"; + /** + * AWS source region to use while initializing the client for service. + */ + public static final String AWS_SOURCE_REGION = "source.region"; + /** + * AWS target endpoint to use while initializing the client for service. + */ + public static final String AWS_TARGET_ENDPOINT = "target.endpoint"; + /** + * AWS target region to use while initializing the client for service. + */ + public static final String AWS_TARGET_REGION = "target.region"; + /** + * Number of schema versions to replicate from source to target + */ + public static final String REPLICATE_SCHEMA_VERSION_COUNT = "replicateSchemaVersionCount"; + /** + * Default number of schema versions to replicate from source to target + */ + public static final Integer DEFAULT_REPLICATE_SCHEMA_VERSION_COUNT = 10; + /** + * Source Registry Name. + */ + public static final String SOURCE_REGISTRY_NAME = "source.registry.name"; + /** + * Target Registry Name. + */ + public static final String TARGET_REGISTRY_NAME = "target.registry.name"; +} diff --git a/cross-region-replication-converter/src/test/avro/AvroMessage.avsc b/cross-region-replication-converter/src/test/avro/AvroMessage.avsc new file mode 100644 index 00000000..4a784cd6 --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/AvroMessage.avsc @@ -0,0 +1,944 @@ +{ + "type" : "record", + "name" : "AvroMessage", + "namespace" : "io.test.avro.core", + "fields" : [ { + "name" : "payload", + "type" : [ "null", { + "type" : "record", + "name" : "Event", + "namespace" : "io.test.trade.v1.order", + "fields" : [ { + "name" : "state", + "type" : { + "type" : "record", + "name" : "State", + "fields" : [ { + "name" : "orderId", + "type" : { + "type" : "record", + "name" : "Id", + "namespace" : "io.test.trade.v1", + "doc" : "Id of an order or position.", + "fields" : [ { + "name" : "source", + "type" : { + "type" : "enum", + "name" : "Source", + "symbols" : [ "ORDER_SERVER", "CLIENT", "UNIVERSE", "L2", "L2_CHAIN", "EXCHANGE", "UNIVERSE_ATTR", "UNDEFINED" ] + } + }, { + "name" : "value", + "type" : { + "type" : "string", + "avro.java.string" : "String" + } + } ] + }, + "doc" : "The ID to reference this order." + }, { + "name" : "accountId", + "type" : { + "type" : "record", + "name" : "Id", + "namespace" : "io.test.trade.v1.common.account", + "fields" : [ { + "name" : "value", + "type" : { + "type" : "string", + "avro.java.string" : "String" + } + } ] + }, + "doc" : "The account that the order is associated to." + }, { + "name" : "allocation", + "type" : { + "type" : "record", + "name" : "Allocation", + "namespace" : "io.test.trade.v1.common", + "fields" : [ { + "name" : "direction", + "type" : { + "type" : "enum", + "name" : "Direction", + "symbols" : [ "BUY", "SELL" ] + } + }, { + "name" : "size", + "type" : { + "type" : "record", + "name" : "Size", + "fields" : [ { + "name" : "value", + "type" : "double" + } ] + } + }, { + "name" : "displaySize", + "type" : "Size", + "doc" : "Size used for presentation and external reporting purposes. Note: Margining, Profit/Loss calculation, exposure, etc., should multiply this size with lotSize for calculations." + }, { + "name" : "displaySizeUnit", + "type" : { + "type" : "enum", + "name" : "DisplaySizeUnit", + "symbols" : [ "SHARES", "CONTRACTS", "AMOUNT_PER_POINTS" ] + }, + "doc" : "Define how the displaySize is expressed." + }, { + "name" : "lotSize", + "type" : "double", + "doc" : "Defined on the instrument. Clients on spread-bet accounts use lot size of 1, while CFD and StockBroking clients use lot size configured on the instrument. Dealers can book orders on any client account with a lot size of 1. Hedge accounts have different rules - they are generally booked in lots with lotSize 1, except equities (and equity options)" + }, { + "name" : "currency", + "type" : { + "type" : "record", + "name" : "ISOCurrency", + "fields" : [ { + "name" : "value", + "type" : { + "type" : "string", + "avro.java.string" : "String" + } + } ] + }, + "doc" : "Currency of the order/position" + } ] + }, + "doc" : "Details of the allocation of this order, size/amount, currency and direction." + }, { + "name" : "instrument", + "type" : { + "type" : "record", + "name" : "Instrument", + "namespace" : "io.test.trade.v1.common", + "fields" : [ { + "name" : "bookingCodeType", + "type" : { + "type" : "enum", + "name" : "BookingCodeType", + "symbols" : [ "EPIC", "ISIN_AND_CURRENCY" ] + }, + "doc" : "Indicates if the booking was made using an ISIN or EPIC. If EPIC, unique instrument identifier is the EPIC. If ISIN_AND_CURRENCY, the unique identifier is ISIN and CURRENCY. In this last case EPIC is also set for internal usage." + }, { + "name" : "epic", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "doc" : "This field is populated for all booking operations and it represents the id of the instrument on which the booking was made. Note: This field will probably be made optional when contract-ids are introduced" + }, { + "name" : "isin", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "doc" : "ISIN is populated for stock-broking deals", + "default" : null + }, { + "name" : "level", + "type" : { + "type" : "record", + "name" : "Level", + "fields" : [ { + "name" : "value", + "type" : "double" + } ] + }, + "doc" : "The level at which the position is booked. This level is used for pnl, margining, auto-hedge, exposure calculation, and other such purposes. This level should always be displayLevel multiplied by scaling factor, however, there are a few UV based flows where that constraint doesn't hold." + }, { + "name" : "displayLevel", + "type" : "Level", + "doc" : "The level displayed in the front-end. Note: PnL calculation, auto-hedge and other such operations should multiply by scaling factor." + }, { + "name" : "scalingFactor", + "type" : "double", + "doc" : "The scaling factor used for booking this position." + }, { + "name" : "instrumentType", + "type" : [ "null", { + "type" : "enum", + "name" : "InstrumentType", + "symbols" : [ "SHARES", "CURRENCIES", "INDICES", "BINARY", "FAST_BINARY", "COMMODITIES", "RATES", "OPTIONS_SHARES", "OPTIONS_CURRENCIES", "OPTIONS_INDICES", "OPTIONS_COMMODITIES", "OPTIONS_RATES", "BUNGEE_SHARES", "BUNGEE_CURRENCIES", "BUNGEE_INDICES", "BUNGEE_COMMODITIES", "BUNGEE_RATES", "CAPPED_BUNGEE", "TEST_MARKETS", "SPORTS", "SECTORS" ] + } ], + "doc" : "Type of the instrument on which the booking is made. This field is made optional to accommodate positions missing instrument type for some reason (very old positions, UV based legacy flows, etc)", + "default" : null + }, { + "name" : "marketName", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "scalingFactorOnInstrumentIfDifferent", + "type" : [ "null", "double" ], + "doc" : "If the scaling factor on the instrument is different from the scaling factor used to book this position then this field carries this new scaling factor. This field is used by a trade anomaly report (maintained by XCON)", + "default" : null + } ] + }, + "doc" : "Details on booked level and instrument information" + }, { + "name" : "timestamps", + "type" : { + "type" : "record", + "name" : "Timestamps", + "namespace" : "io.test.trade.v1.common", + "doc" : "See http://test.io/wiki/Position+History+Tactical+Fixes", + "fields" : [ { + "name" : "created", + "type" : { + "type" : "record", + "name" : "UTCTimestamp", + "fields" : [ { + "name" : "value", + "type" : "long" + } ] + }, + "doc" : "Timestamp of the trade's creation time" + }, { + "name" : "lastModified", + "type" : [ "null", "UTCTimestamp" ], + "doc" : "Timestamp of the trade's modification time. For RESTATE, this field indicates the timestamp of the restate", + "default" : null + }, { + "name" : "lastEdited", + "type" : [ "null", "UTCTimestamp" ], + "doc" : "Applicable only for RESTATES and specifies the timestamp at which this trade was last edited", + "default" : null + }, { + "name" : "margin", + "type" : [ "null", "UTCTimestamp" ], + "doc" : "Timestamp of the trade's margin time.", + "default" : null + } ] + }, + "doc" : "Timestamps of when the order was created, modified or last edited." + }, { + "name" : "isForceOpen", + "type" : "boolean", + "doc" : "Upon full fill, should this order close positions existing in opposite direction?", + "default" : false + }, { + "name" : "attachedStop", + "type" : [ "null", { + "type" : "record", + "name" : "Stop", + "namespace" : "io.test.trade.v1.common.contingent", + "fields" : [ { + "name" : "value", + "type" : "double", + "doc" : "Stop value can be expressed either as a Level or Distance. Use this field in conjunction with valueType" + }, { + "name" : "valueType", + "type" : { + "type" : "enum", + "name" : "StopValueType", + "symbols" : [ "DISTANCE", "LEVEL" ] + }, + "doc" : "Represents the unit in which the stop value is expressed" + }, { + "name" : "isGuaranteed", + "type" : "boolean", + "default" : false + }, { + "name" : "lrPremium", + "type" : [ "null", "double" ], + "doc" : "This field represents a multiplier to be applied to the trade's size to derive a limited risk fee (LR Fee). The LR fee is a monetary amount and is expressed in the currency of the order.", + "default" : null + }, { + "name" : "trailingStop", + "type" : [ "null", { + "type" : "record", + "name" : "TrailingStop", + "fields" : [ { + "name" : "distance", + "type" : "double", + "default" : 0.0 + }, { + "name" : "increment", + "type" : "double", + "default" : 0.0 + } ] + } ], + "default" : null + }, { + "name" : "orderIds", + "type" : [ "null", { + "type" : "array", + "items" : "io.test.trade.v1.Id" + } ], + "doc" : "Ids identifying this stop.", + "default" : null + } ] + } ], + "doc" : "An attached Stop is a 'stop-loss' order; an instruction to close a position when a certain level is breached, to minimize loss.", + "default" : null + }, { + "name" : "attachedLimit", + "type" : [ "null", { + "type" : "record", + "name" : "Limit", + "namespace" : "io.test.trade.v1.common.contingent", + "fields" : [ { + "name" : "value", + "type" : "double", + "doc" : "Limit value can be expressed either as a Level or Distance. Use this field in conjunction with valueType" + }, { + "name" : "valueType", + "type" : { + "type" : "enum", + "name" : "LimitValueType", + "symbols" : [ "DISTANCE", "LEVEL" ] + }, + "doc" : "Represents the unit in which the limit value is expressed" + }, { + "name" : "orderIds", + "type" : [ "null", { + "type" : "array", + "items" : "io.test.trade.v1.Id" + } ], + "doc" : "Ids identifying this limit.", + "default" : null + } ] + } ], + "doc" : "An attached limit is a 'profit-limit' order; an instruction to close a position when a certain level is breached, to guarantee profit.", + "default" : null + }, { + "name" : "legacyInfo", + "type" : { + "type" : "record", + "name" : "LegacyInfo", + "namespace" : "io.test.trade.v1.common", + "doc" : "Legacy information for retro compatibility purpose. Should not be used in any new service.", + "fields" : [ { + "name" : "uvCurrency", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "doc" : "deprecated field", + "default" : "" + }, { + "name" : "marketCommodity", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "doc" : "deprecated field", + "default" : "" + }, { + "name" : "prompt", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "doc" : "Deprecated field. Represents the period at which the instrument expires. This field is also referred to as 'period' in some legacy messages", + "default" : null + }, { + "name" : "submitOrderType", + "type" : [ "null", { + "type" : "enum", + "name" : "SubmitOrderType", + "symbols" : [ "UNCONTROLLED_OPEN", "UNCONTROLLED_CLOSE", "CONTROLLED_OPEN", "CONTROLLED_CLOSE", "UNCONTROLLED_OPEN_WITH_STOP", "UNCONTROLLED_CLOSE_WITH_STOP", "MANUAL_POSITION_DELETE", "OPEN_WITH_EXPIRY_STOP" ] + } ], + "doc" : "deprecated field", + "default" : null + }, { + "name" : "requestType", + "type" : [ "null", { + "type" : "enum", + "name" : "RequestType", + "symbols" : [ "AMEND_ORDER", "FINANCE_ORDER", "CFD_ORDER", "PHYSICAL", "UNATTACHED_LIMIT_ORDER", "UNATTACHED_STOP_ORDER", "UNATTACHED_ORDER_DELETE", "UNATTACHED_ORDER_FILL", "UNATTACHED_BUFFER_LIMITS", "UNATTACHED_BUFFER_LIMITS_DELETE", "MARKET_ORDER" ] + } ], + "doc" : "deprecated field", + "default" : null + }, { + "name" : "exchangeRateEpic", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "doc" : "Deprecated field. This field represents the fx rate epic mapped to the trade's currency.", + "default" : null + } ] + }, + "doc" : "Legacy attributes, a hang over from UV, eg market commod." + }, { + "name" : "channel", + "type" : { + "type" : "record", + "name" : "Channel", + "namespace" : "io.test.trade.v1.common", + "fields" : [ { + "name" : "value", + "type" : { + "type" : "string", + "avro.java.string" : "String" + } + } ] + }, + "doc" : "The channel though which the order was placed , eg. WEB, L2. This will not change if the order is amended. For this, see channel in change info." + }, { + "name" : "expiry", + "type" : { + "type" : "record", + "name" : "Expiry", + "namespace" : "io.test.trade.v1.order.common", + "fields" : [ { + "name" : "timeInForce", + "type" : { + "type" : "enum", + "name" : "TimeInForce", + "namespace" : "io.test.trade.v1.common", + "symbols" : [ "DAY", "GOOD_TILL_CANCEL", "AT_THE_OPENING", "IMMEDIATE_OR_CANCEL", "FILL_OR_KILL", "GOOD_TILL_CROSSING", "GOOD_TILL_DATE", "AT_THE_CLOSE", "DAY_ALL_SESSIONS" ] + } + }, { + "name" : "goodTillDateTimestamp", + "type" : [ "null", "io.test.trade.v1.common.UTCTimestamp" ], + "default" : null + } ] + }, + "doc" : "The date/time of when this order expires." + }, { + "name" : "accountAttributes", + "type" : { + "type" : "record", + "name" : "Attributes", + "namespace" : "io.test.trade.v1.common.account", + "fields" : [ { + "name" : "accountProduct", + "type" : [ "null", { + "type" : "enum", + "name" : "Product", + "symbols" : [ "SPREAD_BET", "CFD", "PHYSICAL" ] + } ], + "default" : null + }, { + "name" : "locale", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "doc" : "Represents the locale of the account, such as en_gb. It is highly likely that this field will be removed in the future." + }, { + "name" : "powerOfAttorneyName", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "doc" : "The POA name specified on the order that booked this position.", + "default" : null + }, { + "name" : "convertOnCloseCurrency", + "type" : [ "null", "io.test.trade.v1.common.ISOCurrency" ], + "doc" : "Retrieved from the convert on close information stored on this position. Note: Only populated if convert-on-close is applicable for this position.", + "default" : null + }, { + "name" : "currency", + "type" : [ "null", "io.test.trade.v1.common.ISOCurrency" ], + "doc" : "Account's currency in ISO format", + "default" : null + } ] + }, + "doc" : "Account attributes such as convert on close details and Power Of Attorney name." + }, { + "name" : "dmaOrder", + "type" : [ "null", { + "type" : "record", + "name" : "Order", + "namespace" : "io.test.trade.v1.common.dma", + "fields" : [ { + "name" : "pseudoPositionId", + "type" : [ "null", "io.test.trade.v1.Id" ], + "doc" : "Represents ID of position created by a partially filled DMA order.", + "default" : null + }, { + "name" : "orderType", + "type" : { + "type" : "enum", + "name" : "OrderType", + "symbols" : [ "MARKET", "LIMIT", "STOP", "STOP_LIMIT", "MARKET_ON_CLOSE", "WITH_OR_WITHOUT", "LIMIT_OR_BETTER", "LIMIT_WITH_OR_WITHOUT", "ON_BASIS", "ON_CLOSE", "LIMIT_ON_CLOSE", "FOREX_MARKET", "PREVIOUSLY_QUOTED", "PREVIOUSLY_INDICATED", "FOREX_LIMIT", "PEGGED", "TRADE_REPORT", "FAST_BINARY", "UNKNOWN" ] + } + }, { + "name" : "timeInForce", + "type" : "io.test.trade.v1.common.TimeInForce" + }, { + "name" : "originalSize", + "type" : "io.test.trade.v1.common.Size", + "doc" : "The original size on a DMA working order. This is in display terms and does not include lotSize" + }, { + "name" : "fills", + "type" : [ "null", { + "type" : "record", + "name" : "Fills", + "fields" : [ { + "name" : "aggregatedFill", + "type" : [ "null", { + "type" : "array", + "items" : { + "type" : "record", + "name" : "AggregatedFill", + "doc" : "Aggregated information of fills received per hedge account", + "fields" : [ { + "name" : "hedgeAccountId", + "type" : "io.test.trade.v1.common.account.Id" + }, { + "name" : "averageLevel", + "type" : "io.test.trade.v1.common.Level", + "doc" : "A volume-weighted-average level of all fills originating from this hedge account" + }, { + "name" : "totalSize", + "type" : "io.test.trade.v1.common.Size", + "doc" : "Total size of all fills received from this hedge account" + }, { + "name" : "averageExchangeFee", + "type" : "double", + "doc" : "The fee is expressed in account's currency." + } ] + } + } ], + "doc" : "A collection of DMA fills aggregated per hedge account", + "default" : null + }, { + "name" : "updateType", + "type" : { + "type" : "enum", + "name" : "FillsUpdateType", + "symbols" : [ "ADD", "COPY", "DELETE_ALL" ] + }, + "default" : "COPY" + }, { + "name" : "nextWorkingOrderId", + "type" : [ "null", "io.test.trade.v1.Id" ], + "default" : null + } ] + } ], + "default" : null + }, { + "name" : "isDMAInteractable", + "type" : "boolean", + "default" : true + }, { + "name" : "executionPricePreference", + "type" : [ "null", { + "type" : "enum", + "name" : "ExecType", + "symbols" : [ "ASK", "BID" ] + } ], + "doc" : "Called executionInstruction in current schema. This field represents DMA FX Stop Order execution price preference, could be either empty, ASK(0) or BID(9) and indicates whether one's order gets executed closer to the Bid or Ask side compared to the specified order direction.", + "default" : null + }, { + "name" : "uvOrderId", + "type" : [ "null", "io.test.trade.v1.Id" ], + "default" : null + }, { + "name" : "isPseudoPosition", + "type" : "boolean", + "doc" : "Is this position a partial fill for a DMA order?", + "default" : false + }, { + "name" : "nextPseudoPositionId", + "type" : [ "null", "io.test.trade.v1.Id" ], + "doc" : "In a DMA amend scenario, the id of a pseudo-position changes and this field indicates the new pseudo position id.", + "default" : null + } ] + } ], + "doc" : "DMA order attributes such as order type", + "default" : null + }, { + "name" : "additionalIds", + "type" : [ "null", { + "type" : "array", + "items" : "io.test.trade.v1.Id" + } ], + "doc" : "Additional ids used to reference this order.", + "default" : null + }, { + "name" : "stockBrokingAttributes", + "type" : [ "null", { + "type" : "record", + "name" : "Attributes", + "namespace" : "io.test.trade.v1.order.stockbroking", + "fields" : [ { + "name" : "settlementDate", + "type" : [ "null", "io.test.trade.v1.common.UTCTimestamp" ], + "default" : null + }, { + "name" : "tradeDate", + "type" : [ "null", "io.test.trade.v1.common.UTCTimestamp" ], + "default" : null + }, { + "name" : "charges", + "type" : [ "null", { + "type" : "record", + "name" : "Charges", + "fields" : [ { + "name" : "commission", + "type" : { + "type" : "record", + "name" : "Money", + "namespace" : "io.test.trade.v1.common", + "fields" : [ { + "name" : "currency", + "type" : "ISOCurrency" + }, { + "name" : "amount", + "type" : "double" + } ] + } + }, { + "name" : "physicalCharges", + "type" : [ "null", { + "type" : "array", + "items" : { + "type" : "record", + "name" : "Charge", + "fields" : [ { + "name" : "code", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "default" : "" + }, { + "name" : "name", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "default" : "" + }, { + "name" : "amount", + "type" : "io.test.trade.v1.common.Money" + }, { + "name" : "rate", + "type" : "double" + }, { + "name" : "threshold", + "type" : "double" + } ] + } + } ], + "default" : null + } ] + } ], + "default" : null + }, { + "name" : "reservedCash", + "type" : [ "null", "io.test.trade.v1.common.Money" ], + "default" : null + } ] + } ], + "doc" : "Stock Broking specific attributes such as settlement date and trade date.", + "default" : null + }, { + "name" : "commissionInstructions", + "type" : [ "null", { + "type" : "record", + "name" : "Instructions", + "namespace" : "io.test.trade.v1.common.commission", + "fields" : [ { + "name" : "bypasses", + "type" : [ "null", { + "type" : "record", + "name" : "Bypasses", + "fields" : [ { + "name" : "legacyCRPremium", + "type" : "boolean", + "doc" : "For guaranteed stops, should bypass reserving LR Premium fee", + "default" : false + }, { + "name" : "commission", + "type" : "boolean", + "doc" : "Should commission be bypassed", + "default" : false + }, { + "name" : "charges", + "type" : "boolean", + "doc" : "Should charges be bypassed", + "default" : false + }, { + "name" : "consideration", + "type" : "boolean", + "doc" : "Should consideration based fee be bypassed", + "default" : false + } ] + } ], + "default" : null + }, { + "name" : "overrideType", + "type" : [ "null", { + "type" : "enum", + "name" : "OverrideType", + "doc" : "AMOUNT: This type used when dealer wants to fixed Commission charge in Client's base currency. When the Amount value is Zero, no commission will be charged.\n. WEB_RATES: This type is used when dealer wants the client's web rates to be used. Otherwise an input from IG Dealer will cause the phone rates to be used.\nPERCENT: This type is used when dealer wants to supply the percentage rate to be used for commission calculation", + "symbols" : [ "AMOUNT", "PERCENT", "WEB_RATES" ] + } ], + "default" : null + }, { + "name" : "rate", + "type" : [ "null", "double" ], + "default" : null + }, { + "name" : "comment", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + } ] + } ], + "doc" : "instructions of which changes to bypass or override.", + "default" : null + }, { + "name" : "additionalLeg", + "type" : [ "null", { + "type" : "record", + "name" : "AdditionalLeg", + "namespace" : "io.test.trade.v1.order.common", + "fields" : [ { + "name" : "instrument", + "type" : "io.test.trade.v1.common.Instrument" + }, { + "name" : "marketCommodity", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "direction", + "type" : "io.test.trade.v1.common.Direction" + }, { + "name" : "averagePrice", + "type" : [ "null", "io.test.trade.v1.common.Level" ], + "default" : null + } ] + } ], + "doc" : "DMA orders on hedge accounts can optionally have an additional leg (instrument) to book the same order.", + "default" : null + }, { + "name" : "profileData", + "type" : [ "null", { + "type" : "record", + "name" : "ProfileData", + "fields" : [ { + "name" : "parentAccountId", + "type" : "io.test.trade.v1.common.account.Id", + "doc" : "For Profile orders this is the reference to the parent account." + }, { + "name" : "parentOrderId", + "type" : "io.test.trade.v1.Id", + "doc" : "For profile orders this will be the parent order id." + } ] + } ], + "doc" : "Profile orders where the order is processed on the parent account and booked against the child accounts.", + "default" : null + }, { + "name" : "lockState", + "type" : [ "null", { + "type" : "record", + "name" : "State", + "namespace" : "io.test.trade.v1.common.lock", + "fields" : [ { + "name" : "idOfLockingDMAOrder", + "type" : "io.test.trade.v1.Id", + "doc" : "Contains id of a DMA order that has locked this position, presumably for explicitly closing this position" + }, { + "name" : "holder", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "source", + "type" : [ "null", { + "type" : "enum", + "name" : "Source", + "symbols" : [ "COM", "DMA", "STOP_MONITOR" ] + } ], + "default" : null + }, { + "name" : "stopMonitorState", + "type" : [ "null", { + "type" : "enum", + "name" : "StopMonitorState", + "symbols" : [ "COM", "DMA", "STOP_MONITOR" ] + } ], + "default" : null + } ] + } ], + "doc" : "Indicates if the order is locked and they type of lock.", + "default" : null + }, { + "name" : "narrative", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "doc" : "Free text for reference. Not used in processing.", + "default" : null + } ] + } + }, { + "name" : "changeInfo", + "type" : { + "type" : "record", + "name" : "Info", + "namespace" : "io.test.trade.v1.order.change", + "fields" : [ { + "name" : "action", + "type" : { + "type" : "enum", + "name" : "Action", + "namespace" : "io.test.trade.v1.common.change", + "symbols" : [ "UPDATE", "DELETE", "NEW", "RESTATE" ] + } + }, { + "name" : "channel", + "type" : [ "null", "io.test.trade.v1.common.Channel" ], + "default" : null + }, { + "name" : "attachedStop", + "type" : [ "null", { + "type" : "record", + "name" : "Stop", + "namespace" : "io.test.trade.v1.common.change.attached", + "fields" : [ { + "name" : "action", + "type" : { + "type" : "enum", + "name" : "Action", + "symbols" : [ "NEW", "DELETED", "UPDATED" ] + } + }, { + "name" : "distance", + "type" : "double" + }, { + "name" : "trailingStop", + "type" : [ "null", { + "type" : "record", + "name" : "TrailingStop", + "fields" : [ { + "name" : "action", + "type" : "Action" + }, { + "name" : "distance", + "type" : "double" + }, { + "name" : "increment", + "type" : "double" + } ] + } ], + "default" : null + } ] + } ], + "default" : null + }, { + "name" : "attachedLimit", + "type" : [ "null", { + "type" : "record", + "name" : "Limit", + "namespace" : "io.test.trade.v1.common.change.attached", + "fields" : [ { + "name" : "action", + "type" : "Action" + }, { + "name" : "distance", + "type" : "double" + } ] + } ], + "default" : null + }, { + "name" : "transactOrderReference", + "type" : [ "null", "io.test.trade.v1.Id" ], + "default" : null + }, { + "name" : "transactTimestamp", + "type" : [ "null", "io.test.trade.v1.common.UTCTimestamp" ], + "default" : null + } ] + } + }, { + "name" : "transaction", + "type" : [ "null", { + "type" : "record", + "name" : "Info", + "namespace" : "io.test.trade.v1.common.transaction", + "fields" : [ { + "name" : "id", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "group", + "type" : [ "null", { + "type" : "record", + "name" : "Group", + "fields" : [ { + "name" : "size", + "type" : "int" + }, { + "name" : "messageIndex", + "type" : "int" + } ] + } ], + "default" : null + } ] + } ], + "default" : null + } ] + } ], + "default" : null + }, { + "name" : "properties", + "type" : [ "null", { + "type" : "map", + "values" : { + "type" : "string", + "avro.java.string" : "String" + }, + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "uuid", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "clientId", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "partitionKey", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "correlationId", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "clusterId", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + } ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/DocTestRecord.avsc b/cross-region-replication-converter/src/test/avro/DocTestRecord.avsc new file mode 100644 index 00000000..b83348ef --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/DocTestRecord.avsc @@ -0,0 +1,20 @@ +{ + "type" : "record", + "name" : "DocTestRecord", + "namespace" : "io.test.avro.doc", + "doc" : "Some record document.", + "fields" : [ { + "name" : "obj", + "type" : { + "type" : "record", + "name" : "DocTestRecord1", + "doc" : "Some nested record document.", + "fields" : [ { + "name" : "data", + "type" : "string", + "doc" : "Some nested record field document." + } ] + }, + "doc" : "Some field document." + } ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/Enum.avsc b/cross-region-replication-converter/src/test/avro/Enum.avsc new file mode 100644 index 00000000..4d24ad60 --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/Enum.avsc @@ -0,0 +1,16 @@ +{ + "namespace": "foo.bar", + "type": "record", + "name": "EnumTest", + "fields": [ + {"name": "testkey", "type": "string"}, + { + "name": "kind", + "type": { + "name": "Kind", + "type": "enum", + "symbols" : ["ONE", "TWO", "THREE"] + } + } + ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/EnumUnion.avsc b/cross-region-replication-converter/src/test/avro/EnumUnion.avsc new file mode 100644 index 00000000..7e90cd0a --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/EnumUnion.avsc @@ -0,0 +1,22 @@ +{ + "type": "record", + "name": "EnumUnion", + "namespace": "com.connect.avro", + "fields": [ + { + "name": "userType", + "type": [ + "null", + { + "type": "enum", + "name": "UserType", + "symbols": [ + "ANONYMOUS", + "REGISTERED" + ] + } + ], + "default": null + } + ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/MultiTypeUnionMessage.avsc b/cross-region-replication-converter/src/test/avro/MultiTypeUnionMessage.avsc new file mode 100644 index 00000000..35aa90b7 --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/MultiTypeUnionMessage.avsc @@ -0,0 +1,45 @@ +{ + "type": "record", + "name": "MultiTypeUnionMessage", + "namespace": "io.test.avro.union", + "fields": [ + { + "name": "CompositeRecord", + "type": [ + "null", + { + "type": "record", + "name": "FirstOption", + "fields": [ + { + "name": "x", + "type": "string" + }, + { + "name": "y", + "type": "long" + } + ] + }, + { + "type": "record", + "name": "SecondOption", + "fields": [ + { + "name": "a", + "type": "string" + }, + { + "name": "b", + "type": "long" + } + ] + }, + { + "type": "array", + "items": "string" + } + ] + } + ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDefault.avsc b/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDefault.avsc new file mode 100644 index 00000000..12b80326 --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDefault.avsc @@ -0,0 +1,45 @@ +{ + "name": "RepeatedTypeWithDefault", + "namespace": "com.rr.avro.test", + "type": "record", + "fields": [ + { + "name": "stringField", + "type": "string", + "default": "field's default" + }, + { + "name": "anotherStringField", + "type": "string" + }, + { + "name": "enumField", + "default": "ONE", + "type": { + "name": "Kind", + "type": "enum", + "symbols" : ["ONE", "TWO", "THREE"] + } + }, + { + "name": "anotherEnumField", + "type": "Kind", + "default": "TWO" + }, + { + "name": "enumFieldWithDiffDefault", + "default": "B", + "type": { + "name": "someKind", + "type": "enum", + "symbols": ["A", "B", "C"], + "default": "A" + } + }, + { + "name": "floatField", + "type": "float", + "default": 9.18 + } + ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDocFull.avsc b/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDocFull.avsc new file mode 100644 index 00000000..5cf02e9b --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDocFull.avsc @@ -0,0 +1,96 @@ +{ + "name": "RepeatedTypeWithDoc", + "namespace": "com.rr.avro.test", + "type": "record", + "doc": "record's doc", + "fields": [ + { + "name": "stringField", + "type": "string", + "doc": "field's doc" + }, + { + "name": "anotherStringField", + "type": "string" + }, + { + "name": "recordField", + "doc": "record field's doc", + "type": { + "name": "NestedRecord", + "type": "record", + "doc": "nested record's doc", + "fields": [ + { + "name": "nestedRecordField", + "doc": "nested record field's doc", + "type": { + "name": "FixedType", + "type": "fixed", + "size": 4 + } + }, + { + "name": "anotherNestedRecordField", + "type": "FixedType" + } + ] + } + }, + { + "name": "anotherRecordField", + "type": "NestedRecord", + "doc": "another record field's doc" + }, + { + "name": "recordFieldWithoutDoc", + "type": "NestedRecord" + }, + { + "name": "doclessRecordField", + "type": { + "name": "DoclessNestedRecord", + "type": "record", + "fields": [ + { + "name": "aField", + "type": "string" + } + ] + } + }, + { + "name": "doclessRecordFieldWithDoc", + "type": "DoclessNestedRecord", + "doc": "docless record field's doc" + }, + { + "name": "enumField", + "doc": "enum field's doc", + "type": { + "name": "Kind", + "type": "enum", + "doc": "enum's doc", + "symbols" : ["ONE", "TWO", "THREE"] + } + }, + { + "name": "anotherEnumField", + "type": "Kind", + "doc": "another enum field's doc" + }, + { + "name": "doclessEnumField", + "type": "Kind" + }, + { + "name": "diffEnumField", + "type": { + "name": "anotherKind", + "type": "enum", + "doc": "diffEnum's doc", + "symbols": ["A", "B", "C"] + } + } + ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverterTest.java b/cross-region-replication-converter/src/test/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverterTest.java new file mode 100644 index 00000000..1da2ebf0 --- /dev/null +++ b/cross-region-replication-converter/src/test/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverterTest.java @@ -0,0 +1,461 @@ +package com.amazonaws.services.crossregion.schemaregistry.kafkaconnect; + +import com.amazonaws.services.schemaregistry.common.Schema; +import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryDeserializerImpl; +import com.amazonaws.services.schemaregistry.exception.AWSSchemaRegistryException; +import com.amazonaws.services.schemaregistry.exception.GlueSchemaRegistryIncompatibleDataException; +import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistrySerializerImpl; +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.errors.DataException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.services.glue.model.DataFormat; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; + +/** + * Unit tests for testing RegisterSchema class. + */ +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) + +public class AWSGlueCrossRegionSchemaReplicationConverterTest { + @Mock + private AwsCredentialsProvider credProvider; + @Mock + private GlueSchemaRegistryDeserializerImpl deserializer; + @Mock + private GlueSchemaRegistrySerializerImpl serializer; + private final static byte[] ENCODED_DATA = new byte[] { 8, 9, 12, 83, 82 }; + private final static byte[] USER_DATA = new byte[] { 12, 83, 82 }; + private static final String testTopic = "User-Topic"; + private AWSGlueCrossRegionSchemaReplicationConverter converter; + + byte[] genericBytes = new byte[] {3, 0, -73, -76, -89, -16, -100, -106, 78, 74, -90, -121, -5, + 93, -23, -17, 12, 99, 10, 115, 97, 110, 115, 97, -58, 1, 6, 114, 101, 100}; + byte[] avroBytes = new byte[] {3, 0, 84, 24, 47, -109, 37, 124, 74, 77, -100, + -98, -12, 118, 41, 32, 57, -66, 30, 101, 110, 116, 101, 114, 116, 97, 105, 110, 109, 101, 110, + 116, 95, 50, 0, 0, 0, 0, 0, 0, 20, 64}; + byte[] jsonBytes = new byte[] {3, 0, -73, -76, -89, -16, -100, -106, 78, 74, -90, -121, -5, 93, -23, -17, 12, 99, 123, 34, + 102, 105, 114, 115, 116, 78, 97, 109, 101, 34, 58, 34, 74, 111, 104, 110, 34, 44, 34, 108, 97, + 115, 116, 78, 97, 109, 101, 34, 58, 34, 68, 111, 101, 34, 44, 34, 97, 103, 101, 34, 58, 50, 49, + 125}; + byte[] protobufBytes = "foo".getBytes(StandardCharsets.UTF_8); + + @BeforeEach + void setUp() { + converter = new AWSGlueCrossRegionSchemaReplicationConverter(credProvider, deserializer, serializer); + } + + /** + * Test for Converter config method. + */ + @Test + public void testConverter_configure() { + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + converter.configure(getTestProperties(), false); + assertNotNull(converter); + assertNotNull(converter.getCredentialsProvider()); + assertNotNull(converter.getSerializer()); + assertNotNull(converter.getDeserializer()); + assertNotNull(converter.isKey()); + } + + /** + * Test for Converter when source region config is not provided. + */ + @Test + public void testConverter_sourceRegionNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoSourceRegionProperties(), false)); + assertEquals("Source Region is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when source registry config is not provided. + */ + @Test + public void testConverter_sourceRegistryNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoSourceRegistryProperties(), false)); + assertEquals("Source Registry is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when source endpoint config is not provided. + */ + @Test + public void testConverter_sourceEndpointNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoSourceEndpointProperties(), false)); + assertEquals("Source Endpoint is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when target region config is not provided. + */ + @Test + public void testConverter_targetRegionNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoTargetRegionProperties(), false)); + assertEquals("Target Region is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when source registry config is not provided. + */ + @Test + public void testConverter_targetRegistryNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoTargetRegistryProperties(), false)); + assertEquals("Target Registry is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when source endpoint config is not provided. + */ + @Test + public void testConverter_targetEndpointNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoTargetEndpointProperties(), false)); + assertEquals("Target Endpoint is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when no target specific config is provided + */ + @Test + public void testConverter_noTargetDetails_Succeeds(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + converter.configure(getPropertiesNoTargetDetails(), false); + assertNotNull(converter.getSerializer()); + } + + /** + * Test Converter when it returns null given the input value is null. + */ + @Test + public void testConverter_fromConnectData_returnsByte0() { + Struct expected = createStructRecord(); + assertNull(converter.fromConnectData(testTopic, expected.schema(), null)); + } + + /** + * Test Converter when serializer throws exception with Avro schema. + */ + @Test + public void testConverter_fromConnectData_serializer_avroSchema_throwsException() { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.AVRO.name(), "schemaFoo"); + Struct expected = createStructRecord(); + doReturn(USER_DATA) + .when(deserializer).getData(genericBytes); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + when(serializer.encode(testTopic, SCHEMA_REGISTRY_SCHEMA, USER_DATA)).thenThrow(new AWSSchemaRegistryException()); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when the deserializer throws exception with Avro schema. + */ + @Test + public void testConverter_fromConnectData_deserializer_avroSchema_throwsException() { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.AVRO.name(), "schemaFoo"); + Struct expected = createStructRecord(); + when((deserializer).getData(genericBytes)).thenThrow(new AWSSchemaRegistryException()); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode(null, SCHEMA_REGISTRY_SCHEMA, USER_DATA); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when Avro schema is replicated. + */ + @Test + public void testConverter_fromConnectData_avroSchema_succeeds() { + String schemaDefinition = "{\"namespace\":\"com.amazonaws.services.schemaregistry.serializers.avro\",\"type\":\"record\",\"name\":\"payment\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"id_6\",\"type\":\"double\"}]}"; + Schema testSchema = new Schema(schemaDefinition, DataFormat.AVRO.name(), testTopic); + Struct expected = createStructRecord(); + doReturn(genericBytes). + when(deserializer).getData(avroBytes); + doReturn(testSchema). + when(deserializer).getSchema(avroBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode(testTopic, testSchema, genericBytes); + assertEquals(converter.fromConnectData(testTopic, expected.schema(), avroBytes), ENCODED_DATA); + } + + /** + * Test Converter when serializer throws exception with JSON schema. + */ + @Test + public void testConverter_fromConnectData_serializer_jsonSchema_throwsException() { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.JSON.name(), "schemaFoo"); + Struct expected = createStructRecord(); + doReturn(USER_DATA) + .when(deserializer).getData(genericBytes); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + when(serializer.encode(testTopic, SCHEMA_REGISTRY_SCHEMA, USER_DATA)).thenThrow(new AWSSchemaRegistryException()); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when the deserializer throws exception with JSON schema. + */ + @Test + public void testConverter_fromConnectData_deserializer_jsonSchema_throwsException() { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.JSON.name(), "schemaFoo"); + Struct expected = createStructRecord(); + when((deserializer).getData(genericBytes)).thenThrow(new AWSSchemaRegistryException()); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode("schemaFoo", SCHEMA_REGISTRY_SCHEMA, USER_DATA); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when JSON schema is replicated. + */ + @Test + public void testConverter_fromConnectData_jsonSchema_succeeds() { + String testSchemaDefinition = "{\"$id\":\"https://example.com/geographical-location.schema.json\"," + + "\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Longitude " + + "and Latitude Values\",\"description\":\"A geographical coordinate.\"," + + "\"required\":[\"latitude\",\"longitude\"],\"type\":\"object\"," + + "\"properties\":{\"latitude\":{\"type\":\"number\",\"minimum\":-90," + + "\"maximum\":90},\"longitude\":{\"type\":\"number\",\"minimum\":-180," + + "\"maximum\":180}},\"additionalProperties\":false}"; + Schema testSchema = new Schema(testSchemaDefinition, DataFormat.JSON.name(), testTopic); + Struct expected = createStructRecord(); + doReturn(genericBytes). + when(deserializer).getData(jsonBytes); + doReturn(testSchema). + when(deserializer).getSchema(jsonBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode(testTopic, testSchema, genericBytes); + assertEquals(converter.fromConnectData(testTopic, expected.schema(), jsonBytes), ENCODED_DATA); + } + + /** + * Test Converter when message without schema is replicated. + */ + @Test + public void testConverter_fromConnectData_noSchema_succeeds() { + Struct expected = createStructRecord(); + when(deserializer.getData(genericBytes)).thenThrow(new GlueSchemaRegistryIncompatibleDataException("No schema in message")); + assertEquals(converter.fromConnectData(testTopic, expected.schema(), genericBytes), genericBytes); + } + + /** + * Test Converter when serializer throws exception with protobuf schema. + */ + @Test + public void testConverter_fromConnectData_serializer_protobufSchema_throwsException() { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.PROTOBUF.name(), "schemaFoo"); + Struct expected = createStructRecord(); + doReturn(USER_DATA) + .when(deserializer).getData(genericBytes); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + when(serializer.encode(testTopic, SCHEMA_REGISTRY_SCHEMA, USER_DATA)).thenThrow(new AWSSchemaRegistryException()); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when the deserializer throws exception with protobuf schema. + */ + @Test + public void testConverter_fromConnectData_deserializer_protobufSchema_throwsException() { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.PROTOBUF.name(), "schemaFoo"); + Struct expected = createStructRecord(); + when((deserializer).getData(genericBytes)).thenThrow(new AWSSchemaRegistryException()); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode("schemaFoo", SCHEMA_REGISTRY_SCHEMA, USER_DATA); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when Protobuf schema is replicated. + */ + @Test + public void getSchema_protobuf_succeeds(){ + Schema testSchema = new Schema("foo", DataFormat.PROTOBUF.name(), testTopic); + Struct expected = createStructRecord(); + doReturn(genericBytes). + when(deserializer).getData(protobufBytes); + doReturn(testSchema). + when(deserializer).getSchema(protobufBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode(testTopic, testSchema, genericBytes); + assertEquals(converter.fromConnectData(testTopic, expected.schema(), protobufBytes), ENCODED_DATA); + } + + /** + * Test toConnectData when IllegalAccessException is thrown. + */ + @Test + public void toConnectData_throwsException(){ + assertThrows(UnsupportedOperationException.class, () -> converter.toConnectData(testTopic, genericBytes)); + } + + /** + * To create a map of configurations without source region. + * + * @return a map of configurations + */ + private Map getNoSourceRegionProperties() { + Map props = new HashMap<>(); + + props.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION, "us-east-1"); + + return props; + } + + /** + * To create a map of configurations without source registry. + * + * @return a map of configurations + */ + private Map getNoSourceRegistryProperties() { + Map props = new HashMap<>(); + + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION, "us-east-1"); + + return props; + } + + /** + * To create a map of configurations without target registry. + * + * @return a map of configurations + */ + private Map getNoTargetRegistryProperties() { + Map props = new HashMap<>(); + + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + + return props; + } + + /** + * To create a map of configurations without source endpoint. + * + * @return a map of configurations + */ + private Map getNoSourceEndpointProperties() { + Map props = new HashMap<>(); + + props.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-west-2"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, "https://test"); + + return props; + } + + /** + * To create a map of configurations without source endpoint. + * + * @return a map of configurations + */ + private Map getNoTargetEndpointProperties() { + Map props = new HashMap<>(); + + props.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-west-2"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.REGISTRY_NAME, "default-registry"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT, "https://test"); + + return props; + } + + /** + * To create a map of configurations without target region. + * + * @return a map of configurations + */ + private Map getNoTargetRegionProperties() { + Map props = new HashMap<>(); + + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-west-2"); + + return props; + } + + /** + * To create a map of configurations without target region, target endpoint and target registry name + * but is replaced by the provided region, endpoint and registry name config. + * + * @return a map of configurations + */ + private Map getPropertiesNoTargetDetails() { + Map props = new HashMap<>(); + + props.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-west-2"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, "https://test"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT, "https://test"); + + return props; + } + + /** + * To create a map of configurations. + * + * @return a map of configurations + */ + private Map getTestProperties() { + Map props = new HashMap<>(); + + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-west-2"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + props.put(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, "https://test"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT, "https://test"); + + return props; + } + + /** + * To create Connect Struct record. + * + * @return Connect Struct + */ + private Struct createStructRecord() { + org.apache.kafka.connect.data.Schema schema = SchemaBuilder.struct() + .build(); + return new Struct(schema); + } +} diff --git a/integration-tests/docker-compose.yml b/integration-tests/docker-compose.yml index a7d133aa..bb5718b3 100644 --- a/integration-tests/docker-compose.yml +++ b/integration-tests/docker-compose.yml @@ -1,10 +1,13 @@ -version: '2' - services: - zookeeper: + zookeeper-kafka: image: 'public.ecr.aws/bitnami/zookeeper:latest' - ports: - - '2181:2182' + container_name: zk-kafka + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + zookeeper-kafka-target: + image: 'public.ecr.aws/bitnami/zookeeper:latest' + container_name: zk-kafka-target environment: - ALLOW_ANONYMOUS_LOGIN=yes @@ -12,15 +15,48 @@ services: image: 'public.ecr.aws/bitnami/kafka:2.8' ports: - '9092:9092' - links: - - zookeeper - container_name: local_kafka + container_name: kafka environment: - KAFKA_BROKER_ID=1 - - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 + - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-kafka:2181 + - ALLOW_PLAINTEXT_LISTENER=yes + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT + - KAFKA_CFG_LISTENERS=CLIENT://:29092,EXTERNAL://:9092 + - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:29092,EXTERNAL://localhost:9092 + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT + - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true + + kafka-target: + image: 'public.ecr.aws/bitnami/kafka:2.8' + ports: + - '9093:9093' + container_name: kafka-target + environment: + - KAFKA_BROKER_ID=2 + - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-kafka-target:2181 + - ALLOW_PLAINTEXT_LISTENER=yes + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT + - KAFKA_CFG_LISTENERS=CLIENT://:29092,EXTERNAL://:9093 + - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka-target:29092,EXTERNAL://localhost:9093 + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT + - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true + + mirrormaker: + build: ./mirrormaker + container_name: mirrormaker + environment: - ALLOW_PLAINTEXT_LISTENER=yes + - SOURCE=kafka:29092 + - DESTINATION=kafka-target:29092 + - TOPICS=^SchemaReplicationTests.* + - ACLS_ENABLED=false + - AWS_PROFILE=default + volumes: + - glue-schema-registry-plugins:/opt/plugins + - "${HOME}/.aws/:/.aws:ro" + depends_on: + - kafka + - kafka-target localstack: container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" @@ -36,3 +72,12 @@ services: volumes: - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - "/var/run/docker.sock:/var/run/docker.sock" + +volumes: + glue-schema-registry-plugins: + driver: local + driver_opts: + type: none + device: ../ + o: bind + diff --git a/integration-tests/mirrormaker/Dockerfile b/integration-tests/mirrormaker/Dockerfile new file mode 100644 index 00000000..ad44e212 --- /dev/null +++ b/integration-tests/mirrormaker/Dockerfile @@ -0,0 +1,39 @@ +FROM public.ecr.aws/bitnami/kafka:2.8 +USER root +RUN install_packages gettext + +RUN mkdir -p /opt/plugins +RUN chown 1234 /opt/plugins + +RUN mkdir -p ~/.aws +RUN chmod 1234 ~/.aws + +# Install the AWS CLI +RUN \ + apt-get update -y && \ + apt-get install -y wget vim python3 unzip python-is-python3 python3-venv && \ + wget "s3.amazonaws.com/aws-cli/awscli-bundle.zip" -O "awscli-bundle.zip" && \ + unzip awscli-bundle.zip && \ + ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws && \ + rm awscli-bundle.zip && \ + rm -rf awscli-bundle + +ADD ./mm2-configs/connect-standalone.properties /opt/mm2/connect-standalone.properties +ADD ./mm2-configs/mirror-checkpoint-connector.properties /opt/mm2/mirror-checkpoint-connector.properties +ADD ./mm2-configs/mirror-heartbeat-connector.properties /opt/mm2/mirror-heartbeat-connector.properties +ADD ./mm2-configs/mirror-source-connector.properties /opt/mm2/mirror-source-connector.properties + +ADD ./run.sh /opt/mm2/run.sh +RUN chmod +x /opt/mm2/run.sh + +RUN mkdir -p /var/run/mm2 +RUN chown 1234 /var/run/mm2 + +ENV TOPICS .* +ENV SOURCE "localhost:9092" +ENV DESTINATION "localhost:9093" +ENV ACLS_ENABLED "false" +ENV AWS_PROFILE "default" + +USER 1234 +CMD /opt/mm2/run.sh diff --git a/integration-tests/mirrormaker/mm2-configs/connect-standalone.properties b/integration-tests/mirrormaker/mm2-configs/connect-standalone.properties new file mode 100644 index 00000000..97e2fed5 --- /dev/null +++ b/integration-tests/mirrormaker/mm2-configs/connect-standalone.properties @@ -0,0 +1,12 @@ +bootstrap.servers=${DESTINATION} + +key.converter=org.apache.kafka.connect.converters.ByteArrayConverter +value.converter=org.apache.kafka.connect.converters.ByteArrayConverter + +key.converter.schemas.enable=true +value.converter.schemas.enable=true + +offset.storage.file.filename=/tmp/connect.offsets +offset.flush.interval.ms=10000 + +plugin.path=/opt/plugins/ diff --git a/integration-tests/mirrormaker/mm2-configs/mirror-checkpoint-connector.properties b/integration-tests/mirrormaker/mm2-configs/mirror-checkpoint-connector.properties new file mode 100644 index 00000000..10792369 --- /dev/null +++ b/integration-tests/mirrormaker/mm2-configs/mirror-checkpoint-connector.properties @@ -0,0 +1,13 @@ +name=mm2-cpc +connector.class=org.apache.kafka.connect.mirror.MirrorCheckpointConnector +clusters=src,dst +source.cluster.alias=src +target.cluster.alias=dst +source.cluster.bootstrap.servers=${SOURCE} +target.cluster.bootstrap.servers=${DESTINATION} +tasks.max=1 +key.converter=org.apache.kafka.connect.converters.ByteArrayConverter +value.converter=org.apache.kafka.connect.converters.ByteArrayConverter +replication.factor=1 +checkpoints.topic.replication.factor=1 +emit.checkpoints.interval.seconds=20 \ No newline at end of file diff --git a/integration-tests/mirrormaker/mm2-configs/mirror-heartbeat-connector.properties b/integration-tests/mirrormaker/mm2-configs/mirror-heartbeat-connector.properties new file mode 100644 index 00000000..176bd923 --- /dev/null +++ b/integration-tests/mirrormaker/mm2-configs/mirror-heartbeat-connector.properties @@ -0,0 +1,13 @@ +name=mm2-hbc +connector.class=org.apache.kafka.connect.mirror.MirrorHeartbeatConnector +clusters=src,dst +source.cluster.alias=src +target.cluster.alias=dst +source.cluster.bootstrap.servers=${SOURCE} +target.cluster.bootstrap.servers=${DESTINATION} +tasks.max=1 +key.converter= org.apache.kafka.connect.converters.ByteArrayConverter +value.converter=org.apache.kafka.connect.converters.ByteArrayConverter +replication.factor=1 +heartbeats.topic.replication.factor=1 +emit.heartbeats.interval.seconds=20 \ No newline at end of file diff --git a/integration-tests/mirrormaker/mm2-configs/mirror-source-connector.properties b/integration-tests/mirrormaker/mm2-configs/mirror-source-connector.properties new file mode 100644 index 00000000..44a304ff --- /dev/null +++ b/integration-tests/mirrormaker/mm2-configs/mirror-source-connector.properties @@ -0,0 +1,36 @@ +name=mm2-msc +connector.class=org.apache.kafka.connect.mirror.MirrorSourceConnector +clusters=src,dst +source.cluster.alias=src +target.cluster.alias=dst +source.cluster.bootstrap.servers=${SOURCE} +target.cluster.bootstrap.servers=${DESTINATION} +topics=${TOPICS} +tasks.max=3 +key.converter=com.amazonaws.services.crossregion.schemaregistry.kafkaconnect.AWSGlueCrossRegionSchemaReplicationConverter +value.converter=com.amazonaws.services.crossregion.schemaregistry.kafkaconnect.AWSGlueCrossRegionSchemaReplicationConverter +replication.factor=1 +offset-syncs.topic.replication.factor=1 +sync.topic.acls.enabled=${ACLS_ENABLED} +refresh.topics.interval.seconds=20 +refresh.groups.interval.seconds=20 +consumer.group.id=mm2-msc-cons +producer.enable.idempotence=true +key.converter.schemas.enable=false +value.converter.schemas.enable=true +key.converter.source.endpoint=https://glue.us-east-1.amazonaws.com +key.converter.source.region=us-east-1 +key.converter.target.endpoint=https://glue.us-east-2.amazonaws.com +key.converter.target.region=us-east-2 +key.converter.source.registry.name=default-registry +key.converter.target.registry.name=default-registry +key.converter.replicateSchemaVersionCount=30 +value.converter.source.endpoint=https://glue.us-east-1.amazonaws.com +value.converter.source.region=us-east-1 +value.converter.target.endpoint=https://glue.us-east-2.amazonaws.com +value.converter.target.region=us-east-2 +value.converter.source.registry.name=default-registry +value.converter.target.registry.name=default-registry +value.converter.replicateSchemaVersionCount=30 +errors.log.enable=true +errors.log.include.messages=true \ No newline at end of file diff --git a/integration-tests/mirrormaker/run.sh b/integration-tests/mirrormaker/run.sh new file mode 100644 index 00000000..8d6c516f --- /dev/null +++ b/integration-tests/mirrormaker/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +envsubst < /opt/mm2/connect-standalone.properties > /var/run/mm2/connect-standalone.properties +envsubst < /opt/mm2/mirror-checkpoint-connector.properties > /var/run/mm2/mirror-checkpoint-connector.properties +envsubst < /opt/mm2/mirror-heartbeat-connector.properties > /var/run/mm2/mirror-heartbeat-connector.properties +envsubst < /opt/mm2/mirror-source-connector.properties > /var/run/mm2/mirror-source-connector.properties + +/opt/bitnami/kafka/bin/connect-standalone.sh \ + /var/run/mm2/connect-standalone.properties \ + /var/run/mm2/mirror-heartbeat-connector.properties \ + /var/run/mm2/mirror-checkpoint-connector.properties \ + /var/run/mm2/mirror-source-connector.properties \ No newline at end of file diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 05ea2450..57f8410e 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -66,6 +66,11 @@ schema-registry-serde ${parent.version} + + software.amazon.glue + schema-registry-cross-region-kafkaconnect-converter + ${parent.version} + software.amazon.glue schema-registry-kafkastreams-serde diff --git a/integration-tests/run-local-tests.sh b/integration-tests/run-local-tests.sh old mode 100644 new mode 100755 index 66c797cf..a4ba9218 --- a/integration-tests/run-local-tests.sh +++ b/integration-tests/run-local-tests.sh @@ -115,9 +115,10 @@ cleanUpConnectFiles() { cleanUpDockerResources || true # Start Kafka using docker command asynchronously -docker-compose up --no-attach localstack & -sleep 10 -## Run mvn tests for Kafka and Kinesis Platforms +docker compose up --no-attach localstack & +## Pause to allow docker build to complete +sleep 60 +## Run mvn tests for Kafka, Kinesis Platforms and Schema Replication cd .. && mvn --file integration-tests/pom.xml verify -Psurefire -X && cd integration-tests cleanUpDockerResources @@ -131,7 +132,7 @@ downloadMongoDBConnector copyGSRConverters runConnectTests() { - docker-compose up --no-attach localstack & + docker compose up --no-attach localstack & setUpMongoDBLocal startKafkaConnectTasks ${1} echo "Waiting for Sink task to pick up data.." diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/properties/GlueSchemaRegistryConnectionProperties.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/properties/GlueSchemaRegistryConnectionProperties.java index aff3527b..548363ba 100644 --- a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/properties/GlueSchemaRegistryConnectionProperties.java +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/properties/GlueSchemaRegistryConnectionProperties.java @@ -19,4 +19,8 @@ public interface GlueSchemaRegistryConnectionProperties { // Glue Service Endpoint String REGION = Regions.getCurrentRegion() == null ? "us-east-2" : Regions.getCurrentRegion().getName().toLowerCase(); String ENDPOINT = String.format("https://glue.%s.amazonaws.com", REGION); + String SRC_REGION = Regions.getCurrentRegion() == null ? "us-east-1" : Regions.getCurrentRegion().getName().toLowerCase(); + String SRC_ENDPOINT = String.format("https://glue.%s.amazonaws.com", SRC_REGION); + String DEST_REGION = "us-east-2"; + String DEST_ENDPOINT = String.format("https://glue.%s.amazonaws.com", DEST_REGION); } diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/AWSGlueCrossRegionSchemaReplicationIntegrationTest.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/AWSGlueCrossRegionSchemaReplicationIntegrationTest.java new file mode 100644 index 00000000..8d8e3bb8 --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/AWSGlueCrossRegionSchemaReplicationIntegrationTest.java @@ -0,0 +1,337 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +import com.amazonaws.services.crossregion.schemaregistry.kafkaconnect.AWSGlueCrossRegionSchemaReplicationConverter; +import com.amazonaws.services.crossregion.schemaregistry.kafkaconnect.SchemaReplicationSchemaRegistryConstants; +import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryKafkaDeserializer; +import com.amazonaws.services.schemaregistry.integrationtests.generators.*; +import com.amazonaws.services.schemaregistry.integrationtests.properties.GlueSchemaRegistryConnectionProperties; +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import com.amazonaws.services.schemaregistry.utils.AvroRecordType; +import com.amazonaws.services.schemaregistry.utils.ProtobufMessageType; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Message; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.Compatibility; +import software.amazon.awssdk.services.glue.model.DataFormat; +import software.amazon.awssdk.services.glue.model.DeleteSchemaRequest; +import software.amazon.awssdk.services.glue.model.SchemaId; + +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * The test class for schema replication related tests for Glue Schema Registry + */ +@Slf4j +public class AWSGlueCrossRegionSchemaReplicationIntegrationTest { + private static final String SRC_CLUSTER_ALIAS = "src"; + private static final String TOPIC_NAME_PREFIX = "SchemaReplicationTests"; + private static final String TOPIC_NAME_PREFIX_CONVERTER = "SchemaRegistryTests"; + private static final String SCHEMA_REGISTRY_SRC_ENDPOINT_OVERRIDE = GlueSchemaRegistryConnectionProperties.SRC_ENDPOINT; + private static final String SCHEMA_REGISTRY_DEST_ENDPOINT_OVERRIDE = GlueSchemaRegistryConnectionProperties.DEST_ENDPOINT; + private static final String SRC_REGION = GlueSchemaRegistryConnectionProperties.SRC_REGION; + private static final String DEST_REGION = GlueSchemaRegistryConnectionProperties.DEST_REGION; + private static final String RECORD_TYPE = "GENERIC_RECORD"; + private static final List COMPATIBILITIES = Compatibility.knownValues() + .stream() + .filter(c -> c.toString().equals("NONE") + || c.toString().equals("BACKWARD")) + .collect(Collectors.toList()); + private static LocalKafkaClusterHelper srcKafkaClusterHelper = new LocalKafkaClusterHelper(); + private static LocalKafkaClusterHelper destKafkaClusterHelper = new LocalKafkaClusterHelper(); + private static AwsCredentialsProvider awsCredentialsProvider = DefaultCredentialsProvider.builder() + .build(); + private static List schemasToCleanUp = new ArrayList<>(); + private final TestDataGeneratorFactory testDataGeneratorFactory = new TestDataGeneratorFactory(); + + private static Stream testArgumentsProvider() { + Stream.Builder argumentBuilder = Stream.builder(); + for (DataFormat dataFormat : DataFormat.knownValues()) { + //TODO: Remove if logic + //if (dataFormat == DataFormat.PROTOBUF) { + for (Compatibility compatibility : COMPATIBILITIES) { + //if (compatibility == Compatibility.BACKWARD) { + for (AWSSchemaRegistryConstants.COMPRESSION compression : + AWSSchemaRegistryConstants.COMPRESSION.values()) { + argumentBuilder.add(Arguments.of(dataFormat, RECORD_TYPE, compatibility, compression)); + } + //} + } + //} + } + return argumentBuilder.build(); + } + + private static Pair createAndGetKafkaHelper(String topicNamePrefix) throws Exception { + final String topic = String.format("%s-%s-%s", topicNamePrefix, Instant.now() + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ofPattern("yy-MM-dd-HH-mm")), RandomStringUtils.randomAlphanumeric(4)); + + final String srcBootstrapString = srcKafkaClusterHelper.getSrcClusterBootstrapString(); + final KafkaHelper kafkaHelper = new KafkaHelper(srcBootstrapString, srcKafkaClusterHelper.getOrCreateCluster()); + kafkaHelper.createTopic(topic, srcKafkaClusterHelper.getNumberOfPartitions(), srcKafkaClusterHelper.getReplicationFactor()); + return Pair.of(topic, kafkaHelper); + } + + @Test + public void testProduceConsumeWithoutSchemaRegistry() throws Exception { + log.info("Starting the test for producing and consuming messages via Kafka ..."); + + final Pair srcKafkaHelperPair = createAndGetKafkaHelper(TOPIC_NAME_PREFIX); + String topic = srcKafkaHelperPair.getKey(); + KafkaHelper srcKafkaHelper = srcKafkaHelperPair.getValue(); + KafkaHelper destKafkaHelper = new KafkaHelper(destKafkaClusterHelper.getDestClusterBootstrapString(), destKafkaClusterHelper.getOrCreateCluster()); + + final int recordsProduced = 20; + srcKafkaHelper.doProduce(topic, recordsProduced); + + //Delay added to allow MM2 copy the data to destination cluster + //before consuming the records from the destination cluster + Thread.sleep(10000); + + ConsumerProperties consumerProperties = ConsumerProperties.builder() + .topicName(String.format("%s.%s",SRC_CLUSTER_ALIAS, topic)) + .build(); + + int recordsConsumed = destKafkaHelper.doConsume(consumerProperties); + log.info("Producing {} records, and consuming {} records", recordsProduced, recordsConsumed); + + assertEquals(recordsConsumed, recordsProduced); + log.info("Finish the test for producing/consuming messages via Kafka."); + } + + @ParameterizedTest + @MethodSource("testArgumentsProvider") + public void testProduceConsumeWithSchemaRegistryForAllThreeDataFormatsWithMM2(final DataFormat dataFormat, + final AvroRecordType avroRecordType, + final Compatibility compatibility) throws Exception { + log.info("Starting the test for producing and consuming {} messages via Kafka ...", dataFormat.name()); + final Pair srcKafkaHelperPair = createAndGetKafkaHelper(TOPIC_NAME_PREFIX); + String topic = srcKafkaHelperPair.getKey(); + KafkaHelper srcKafkaHelper = srcKafkaHelperPair.getValue(); + KafkaHelper destKafkaHelper = new KafkaHelper(destKafkaClusterHelper.getDestClusterBootstrapString(), destKafkaClusterHelper.getOrCreateCluster()); + + TestDataGenerator testDataGenerator = testDataGeneratorFactory.getInstance( + TestDataGeneratorType.valueOf(dataFormat, avroRecordType, compatibility)); + List records = testDataGenerator.createRecords(); + + String schemaName = String.format("%s-%s", topic, dataFormat.name()); + schemasToCleanUp.add(schemaName); + + ProducerProperties producerProperties = ProducerProperties.builder() + .topicName(topic) + .schemaName(schemaName) + .dataFormat(dataFormat.name()) + .compatibilityType(compatibility.name()) + .autoRegistrationEnabled("true") + .build(); + + List> producerRecords = + srcKafkaHelper.doProduceRecords(producerProperties, records); + + //Delay added to allow MM2 copy the data to destination cluster + //before consuming the records from the destination cluster + Thread.sleep(10000); + + ConsumerProperties.ConsumerPropertiesBuilder consumerPropertiesBuilder = ConsumerProperties.builder() + .topicName(String.format("%s.%s",SRC_CLUSTER_ALIAS, topic)); + + consumerPropertiesBuilder.protobufMessageType(ProtobufMessageType.DYNAMIC_MESSAGE.getName()); + consumerPropertiesBuilder.avroRecordType(avroRecordType.getName()); // Only required for the case of AVRO + + List> consumerRecords = destKafkaHelper.doConsumeRecords(consumerPropertiesBuilder.build()); + + assertRecordsEquality(producerRecords, consumerRecords); + log.info("Finished test for producing/consuming {} messages via Kafka.", dataFormat.name()); + } + + @ParameterizedTest + @MethodSource("testArgumentsProvider") + public void testProduceConsumeWithSchemaRegistryForAllThreeDataFormatsWithConverter(final DataFormat dataFormat, + final AvroRecordType avroRecordType, + final Compatibility compatibility) throws Exception { + log.info("Starting the test for producing and consuming {} messages via Kafka ...", dataFormat.name()); + final Pair srcKafkaHelperPair = createAndGetKafkaHelper(TOPIC_NAME_PREFIX_CONVERTER); + String topic = srcKafkaHelperPair.getKey(); + KafkaHelper srcKafkaHelper = srcKafkaHelperPair.getValue(); + + TestDataGenerator testDataGenerator = testDataGeneratorFactory.getInstance( + TestDataGeneratorType.valueOf(dataFormat, avroRecordType, compatibility)); + List records = testDataGenerator.createRecords(); + + String schemaName = String.format("%s-%s", topic, dataFormat.name()); + schemasToCleanUp.add(schemaName); + + ProducerProperties producerProperties = ProducerProperties.builder() + .topicName(topic) + .schemaName(schemaName) + .dataFormat(dataFormat.name()) + .compatibilityType(compatibility.name()) + .autoRegistrationEnabled("true") + .build(); + + List> producerRecords = + srcKafkaHelper.doProduceRecords(producerProperties, records); + + ConsumerProperties.ConsumerPropertiesBuilder consumerPropertiesBuilder = ConsumerProperties.builder() + .topicName(topic); + + consumerPropertiesBuilder.protobufMessageType(ProtobufMessageType.DYNAMIC_MESSAGE.getName()); + consumerPropertiesBuilder.avroRecordType(avroRecordType.getName()); // Only required for the case of AVRO + + List> consumerRecords = srcKafkaHelper.doConsumeRecordsWithByteArrayDeserializer(consumerPropertiesBuilder.build()); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + converter.configure(getTestProperties(), false); + + GlueSchemaRegistryKafkaDeserializer deserializer = new GlueSchemaRegistryKafkaDeserializer( + DefaultCredentialsProvider.builder().build(), + getTestProperties()); + + List consumerRecordsDeserialized = new ArrayList<>(); + + for (ConsumerRecord record: consumerRecords) { + byte[] serializedData = converter.fromConnectData(topic, null, record.value()); + Object deserializedData = deserializer.deserialize(topic, serializedData); + consumerRecordsDeserialized.add(deserializedData); + } + + assertRecordsEqualityV2(producerRecords, consumerRecordsDeserialized); + log.info("Finished test for producing/consuming {} messages via Kafka.", dataFormat.name()); + } + + private Map getTestProperties() { + Map props = new HashMap<>(); + + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION, "us-east-2"); + props.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-2"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + props.put(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.REGISTRY_NAME, "default-registry"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT, "https://glue.us-east-1.amazonaws.com"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT, "https://glue.us-east-2.amazonaws.com"); + props.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, "https://glue.us-east-2.amazonaws.com"); + props.put(SchemaReplicationSchemaRegistryConstants.REPLICATE_SCHEMA_VERSION_COUNT, 100); + props.put(AWSSchemaRegistryConstants.AVRO_RECORD_TYPE, AvroRecordType.GENERIC_RECORD.getName()); + + return props; + } + + @AfterAll + public static void tearDown() throws URISyntaxException { + log.info("Starting Clean-up of schemas created with GSR."); + GlueClient glueClientSrc = GlueClient.builder() + .credentialsProvider(awsCredentialsProvider) + .region(Region.of(SRC_REGION)) + .endpointOverride(new URI(SCHEMA_REGISTRY_SRC_ENDPOINT_OVERRIDE)) + .httpClient(UrlConnectionHttpClient.builder() + .build()) + .build(); + GlueClient glueClientDest = GlueClient.builder() + .credentialsProvider(awsCredentialsProvider) + .region(Region.of(DEST_REGION)) + .endpointOverride(new URI(SCHEMA_REGISTRY_DEST_ENDPOINT_OVERRIDE)) + .httpClient(UrlConnectionHttpClient.builder() + .build()) + .build(); + + for (String schemaName : schemasToCleanUp) { + log.info("Cleaning up schema {}..", schemaName); + DeleteSchemaRequest deleteSchemaRequest = DeleteSchemaRequest.builder() + .schemaId(SchemaId.builder() + .registryName("default-registry") + .schemaName(schemaName) + .build()) + .build(); + + glueClientSrc.deleteSchema(deleteSchemaRequest); + glueClientDest.deleteSchema(deleteSchemaRequest); + } + + log.info("Finished Cleaning up {} schemas created with GSR.", schemasToCleanUp.size()); + } + + private void assertRecordsEquality(List> producerRecords, + List> consumerRecords) { + assertThat(producerRecords.size(), is(equalTo(consumerRecords.size()))); + Map producerRecordsMap = producerRecords.stream() + .collect(Collectors.toMap(ProducerRecord::key, ProducerRecord::value)); + + for (ConsumerRecord consumerRecord : consumerRecords) { + assertThat(producerRecordsMap, hasKey(consumerRecord.key())); + if (consumerRecord.value() instanceof DynamicMessage) { + assertDynamicRecords(consumerRecord, producerRecordsMap); + } else { + assertThat(consumerRecord.value(), is(equalTo(producerRecordsMap.get(consumerRecord.key())))); + } + } + } + + private void assertRecordsEqualityV2(List> inputRecords, + List outputRecords) { + assertEquals(inputRecords.size(), outputRecords.size()); + + for (int i =0; i < inputRecords.size(); i++) { + if (outputRecords.get(i) instanceof DynamicMessage) { + assertDynamicRecords(outputRecords.get(i), inputRecords.get(i).value()); + } else { + assertEquals(inputRecords.get(i).value(), outputRecords.get(i)); + } + } + } + + private void assertDynamicRecords(Object consumerRecord, T producerRecord) { + DynamicMessage consumerDynamicMessage = (DynamicMessage) consumerRecord; + Message producerDynamicMessage = (Message) producerRecord; + //In case of DynamicMessage de-serialization, we cannot equate them to POJO records, + //so we check for their byte equality. + assertThat(consumerDynamicMessage.toByteArray(), is(producerDynamicMessage.toByteArray())); + } + + private void assertDynamicRecords(ConsumerRecord consumerRecord, Map producerRecordsMap) { + DynamicMessage consumerDynamicMessage = (DynamicMessage) consumerRecord.value(); + Message producerDynamicMessage = (Message) producerRecordsMap.get(consumerRecord.key()); + //In case of DynamicMessage de-serialization, we cannot equate them to POJO records, + //so we check for their byte equality. + assertThat(consumerDynamicMessage.toByteArray(), is(producerDynamicMessage.toByteArray())); + } +} diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ConsumerProperties.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ConsumerProperties.java new file mode 100644 index 00000000..84b005af --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ConsumerProperties.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +import com.amazonaws.services.schemaregistry.integrationtests.properties.GlueSchemaRegistryConnectionProperties; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class ConsumerProperties implements GlueSchemaRegistryConnectionProperties { + private String topicName; + private String avroRecordType; + private String protobufMessageType; +} + diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaClusterHelper.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaClusterHelper.java new file mode 100644 index 00000000..70409a40 --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaClusterHelper.java @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +public interface KafkaClusterHelper { + String getOrCreateCluster(); + + String getSrcClusterBootstrapString(); + + String getDestClusterBootstrapString(); + + int getNumberOfPartitions(); + + short getReplicationFactor(); +} \ No newline at end of file diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaHelper.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaHelper.java new file mode 100644 index 00000000..84dab8e4 --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaHelper.java @@ -0,0 +1,297 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryKafkaDeserializer; +import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistryKafkaSerializer; +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; + +import java.time.Duration; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.stream.Collectors; + +@Slf4j +public class KafkaHelper { + private static final Duration CONSUMER_RUNTIME = Duration.ofMillis(10000); + private final String bootstrapBrokers; + private final String clusterArn; + + public KafkaHelper(final String bootstrapString, final String clusterArn) { + this.bootstrapBrokers = bootstrapString; + this.clusterArn = clusterArn; + } + + /** + * Helper function to create test topic + * + * @param topic topic name to be created + * @param numPartitions number of numPartitions + * @param replicationFactor replicationFactor count + * @throws Exception + */ + public void createTopic(final String topic, final int numPartitions, final short replicationFactor) throws Exception { + final Properties properties = new Properties(); + properties.put("bootstrap.servers", bootstrapBrokers); + properties.put("client.id", "gsr-integration-tests"); + + log.info("Creating Kafka topic {} with bootstrap {}...", topic, bootstrapBrokers); + try (AdminClient kafkaAdminClient = AdminClient.create(properties)) { + final NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor); + final CreateTopicsResult createTopicsResult = kafkaAdminClient + .createTopics(Collections.singleton(newTopic)); + createTopicsResult.values().get(topic).get(); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + } + + /** + * Helper function to test producer can send messages + * + * @param topic topic to send messages to + * @param numRecords number of records to be sent + * @throws Exception + */ + public void doProduce(final String topic, final int numRecords) throws Exception { + log.info("Start producing to cluster {} with bootstrap {}...", clusterArn, bootstrapBrokers); + + final Properties properties = getKafkaProducerProperties(); + properties.put("key.serializer", StringSerializer.class.getName()); + properties.put("value.serializer", StringSerializer.class.getName()); + + try (Producer producer = new KafkaProducer<>(properties)) { + for (int i = 0; i < numRecords; i++) { + log.info("Producing record " + i); + producer.send(new ProducerRecord<>(topic, Integer.toString(i), Integer.toString(i))).get(); + } + } + + log.info("Finishing producing messages via Kafka."); + } + + /** + * Helper method to test consumption of records + * + * @param consumerProperties consumerProperties + * @return + */ + public int doConsume(final ConsumerProperties consumerProperties) { + final Properties properties = getKafkaConsumerProperties(consumerProperties, true); + properties.put("key.deserializer", StringDeserializer.class.getName()); + properties.put("value.deserializer", StringDeserializer.class.getName()); + final KafkaConsumer consumer = new KafkaConsumer<>(properties); + return consumeRecords(consumer, consumerProperties.getTopicName()).size(); + } + + /** + * Helper function to produce test AVRO records + * + * @param producerProperties producer properties + * @return list of produced records + */ + public List> doProduceRecords(final ProducerProperties producerProperties, + final List records) throws Exception { + Properties properties = getProducerProperties(producerProperties); + properties.put("key.serializer", StringSerializer.class.getName()); + properties.put("value.serializer", GlueSchemaRegistryKafkaSerializer.class.getName()); + Producer producer = new KafkaProducer<>(properties); + + return produceRecords(producer, producerProperties, records); + } + + /** + * Helper function to test consumption of records + * + * @param + */ + public List> doConsumeRecords(final ConsumerProperties consumerProperties) { + Properties properties = getConsumerProperties(consumerProperties, true); + properties.put("key.deserializer", StringDeserializer.class.getName()); + properties.put("value.deserializer", GlueSchemaRegistryKafkaDeserializer.class.getName()); + + final KafkaConsumer consumer = new KafkaConsumer<>(properties); + return consumeRecords(consumer, consumerProperties.getTopicName()); + } + + /** + * Helper function to test consumption of records using ByteArrayDeserializer + * + * @param + * @return + */ + public List> doConsumeRecordsWithByteArrayDeserializer(final ConsumerProperties consumerProperties) { + Properties properties = getConsumerProperties(consumerProperties, false); + properties.put("key.deserializer", org.apache.kafka.common.serialization.ByteArrayDeserializer.class.getName()); + properties.put("value.deserializer", org.apache.kafka.common.serialization.ByteArrayDeserializer.class.getName()); + + final KafkaConsumer consumer = new KafkaConsumer<>(properties); + return consumeRecordsAsByteArray(consumer, consumerProperties.getTopicName()); + } + + /** + * Helper function to process Kafka Streams + * + * @param producerProperties + */ + + private List> produceRecords(final Producer producer, + final ProducerProperties producerProperties, + final List records) throws Exception { + log.info("Start producing to cluster {} with bootstrap {}...", clusterArn, bootstrapBrokers); + List> producerRecords = new ArrayList<>(); + + for (int i = 0; i < records.size(); i++) { + log.info("Fetching record {} for Kafka: {}", i, (T) records.get(i)); + + final ProducerRecord producerRecord; + + // Verify and use a unique field present in the schema as a key for the producer record. + producerRecord = new ProducerRecord<>(producerProperties.getTopicName(), "message-" + i, (T) records.get(i)); + + producerRecords.add(producerRecord); + producer.send(producerRecord); + Thread.sleep(500); + log.info("Sent {} message {}", producerProperties.getDataFormat(), i); + } + producer.flush(); + log.info("Successfully produced {} messages to a topic called {}", records.size(), producerProperties.getTopicName()); + return producerRecords; + } + + private List> consumeRecords(final KafkaConsumer consumer, + final String topic) { + log.info("Start consuming from cluster {} with bootstrap {} ...", clusterArn, bootstrapBrokers); + + consumer.subscribe(Collections.singleton(topic)); + List> consumerRecords = new ArrayList<>(); + final long now = System.currentTimeMillis(); + while (System.currentTimeMillis() - now < CONSUMER_RUNTIME.toMillis()) { + final ConsumerRecords recordsReceived = consumer.poll(Duration.ofMillis(CONSUMER_RUNTIME.toMillis())); + int i = 0; + for (final ConsumerRecord record : recordsReceived) { + final String key = record.key(); + final T value = record.value(); + log.info("Received message {}: key = {}, value = {}", i, key, value); + consumerRecords.add(record); + i++; + } + } + + consumer.close(); + log.info("Finished consuming messages via Kafka."); + return consumerRecords; + } + + private List> consumeRecordsAsByteArray(final KafkaConsumer consumer, + final String topic) { + log.info("Start consuming from cluster {} with bootstrap {} ...", clusterArn, bootstrapBrokers); + + consumer.subscribe(Collections.singleton(topic)); + List> consumerRecords = new ArrayList<>(); + final long now = System.currentTimeMillis(); + while (System.currentTimeMillis() - now < CONSUMER_RUNTIME.toMillis()) { + final ConsumerRecords recordsReceived = consumer.poll(Duration.ofMillis(CONSUMER_RUNTIME.toMillis())); + for (final ConsumerRecord record : recordsReceived) { + consumerRecords.add(record); + } + } + + consumer.close(); + log.info("Finished consuming messages via Kafka."); + return consumerRecords; + } + + private Properties getProducerProperties(final ProducerProperties producerProperties) { + Properties properties = getKafkaProducerProperties(); + setSchemaRegistrySerializerProperties(properties, producerProperties); + return properties; + } + + private Properties getKafkaProducerProperties() { + Properties properties = new Properties(); + properties.put("bootstrap.servers", bootstrapBrokers); + properties.put("acks", "all"); + properties.put("retries", 0); + properties.put("batch.size", 16384); + properties.put("linger.ms", 1); + properties.put("buffer.memory", 33554432); + properties.put("block.on.buffer.full", false); + properties.put("request.timeout.ms", "1000"); + return properties; + } + + private Properties getConsumerProperties(final ConsumerProperties consumerProperties, boolean shouldAddRegistryDetails) { + Properties properties = getKafkaConsumerProperties(consumerProperties, shouldAddRegistryDetails); + return properties; + } + + private Properties getKafkaConsumerProperties(final ConsumerProperties consumerProperties, boolean shouldAddRegistryDetails) { + Properties properties = new Properties(); + properties.put("bootstrap.servers", bootstrapBrokers); + properties.put("group.id", UUID.randomUUID().toString()); + properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + + if (shouldAddRegistryDetails) { + properties.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, consumerProperties.DEST_ENDPOINT); + properties.put(AWSSchemaRegistryConstants.AWS_REGION, consumerProperties.DEST_REGION); + } + + if(consumerProperties.getAvroRecordType() != null) { + properties.put(AWSSchemaRegistryConstants.AVRO_RECORD_TYPE, consumerProperties.getAvroRecordType()); + } + if(consumerProperties.getProtobufMessageType() != null) { + properties.put(AWSSchemaRegistryConstants.PROTOBUF_MESSAGE_TYPE, + consumerProperties.getProtobufMessageType()); + } + return properties; + } + + private void setSchemaRegistrySerializerProperties(final Properties properties, + final ProducerProperties producerProperties) { + properties.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, producerProperties.SRC_ENDPOINT); + properties.put(AWSSchemaRegistryConstants.AWS_REGION, producerProperties.SRC_REGION); + properties.put(AWSSchemaRegistryConstants.SCHEMA_NAME, producerProperties.getSchemaName()); + properties.put(AWSSchemaRegistryConstants.DATA_FORMAT, producerProperties.getDataFormat()); + properties.put(AWSSchemaRegistryConstants.COMPATIBILITY_SETTING, producerProperties.getCompatibilityType()); + properties.put(AWSSchemaRegistryConstants.SCHEMA_AUTO_REGISTRATION_SETTING, producerProperties.getAutoRegistrationEnabled()); + } + + /** + * Create Config map from the properties Object passed. + * + * @param properties properties of configuration elements. + * @return map of configs. + */ + private Map getMapFromPropertiesFile(Properties properties) { + return new HashMap<>(properties.entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey().toString(), e -> e.getValue()))); + } +} \ No newline at end of file diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/LocalKafkaClusterHelper.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/LocalKafkaClusterHelper.java new file mode 100644 index 00000000..48610bca --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/LocalKafkaClusterHelper.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +public class LocalKafkaClusterHelper implements KafkaClusterHelper { + private static final String FAKE_CLUSTER_ARN = "FAKE_CLUSTER_ARN"; + private static final String SRC_BOOTSTRAP_STRING = "127.0.0.1:9092"; + private static final String DEST_BOOTSTRAP_STRING = "127.0.0.1:9093"; + private static final int NUMBER_OF_PARTITIONS = 1; + private static final short REPLICATION_FACTOR = 1; + + @Override + public String getOrCreateCluster() { + return FAKE_CLUSTER_ARN; + } + + @Override + public String getSrcClusterBootstrapString() { + return SRC_BOOTSTRAP_STRING; + } + + @Override + public String getDestClusterBootstrapString() { + return DEST_BOOTSTRAP_STRING; + } + + @Override + public int getNumberOfPartitions() { + return NUMBER_OF_PARTITIONS; + } + + @Override + public short getReplicationFactor() { + return REPLICATION_FACTOR; + } +} + diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ProducerProperties.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ProducerProperties.java new file mode 100644 index 00000000..0693cc42 --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ProducerProperties.java @@ -0,0 +1,34 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +import com.amazonaws.services.schemaregistry.integrationtests.properties.GlueSchemaRegistryConnectionProperties; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class ProducerProperties implements GlueSchemaRegistryConnectionProperties { + private String topicName; + private String schemaName; + private String dataFormat; + private String compatibilityType; + private String autoRegistrationEnabled; + // Streaming properties + private String inputTopic; + private String outputTopic; + private String recordType; // required only for AVRO or Protobuf case +} + diff --git a/pom.xml b/pom.xml index dbfe18bb..8f438972 100644 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ integration-tests jsonschema-kafkaconnect-converter protobuf-kafkaconnect-converter + cross-region-replication-converter diff --git a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/GlueSchemaRegistryDeserializerImplTest.java b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/GlueSchemaRegistryDeserializerImplTest.java index 66116306..415c2890 100644 --- a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/GlueSchemaRegistryDeserializerImplTest.java +++ b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/deserializers/GlueSchemaRegistryDeserializerImplTest.java @@ -10,14 +10,14 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; import java.nio.ByteBuffer; import java.util.Map; import java.util.UUID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; diff --git a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/serializers/avro/GlueSchemaRegistrySerializationFacadeTest.java b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/serializers/avro/GlueSchemaRegistrySerializationFacadeTest.java index 86935524..a361a67a 100644 --- a/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/serializers/avro/GlueSchemaRegistrySerializationFacadeTest.java +++ b/serializer-deserializer/src/test/java/com/amazonaws/services/schemaregistry/serializers/avro/GlueSchemaRegistrySerializationFacadeTest.java @@ -52,7 +52,6 @@ import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -542,8 +541,6 @@ public void testSerialize_arraysWithCompression_byteArraySizeIsReduced(AWSSchema configs.remove(AWSSchemaRegistryConstants.DATA_FORMAT); } - - /** * Tests registerSchemaVersion method of Serializer with metadata configuration */ @@ -700,26 +697,6 @@ public void testEncode_WhenValidInputIsPassed_EncodesTheBytes(DataFormat dataFor AWSSchemaRegistryConstants.COMPRESSION.NONE, payload); } - @Test - public void testEncode_WhenNonSchemaConformantDataIsPassed_ThrowsException() throws Exception { - - JsonDataWithSchema jsonNonSchemaConformantRecord = RecordGenerator.createNonSchemaConformantJsonData(); - String schemaDefinition = jsonNonSchemaConformantRecord.getSchema(); - byte[] payload = jsonNonSchemaConformantRecord.getPayload().getBytes(StandardCharsets.UTF_8); - - final DataFormat dataFormat = DataFormat.JSON; - com.amazonaws.services.schemaregistry.common.Schema schema = - new com.amazonaws.services.schemaregistry.common.Schema(schemaDefinition, dataFormat.name(), TEST_SCHEMA); - - GlueSchemaRegistrySerializationFacade glueSchemaRegistrySerializationFacade = - createGlueSerializationFacade(configs, mockSchemaByDefinitionFetcher); - - //Test subject - Exception ex = assertThrows(AWSSchemaRegistryException.class, - () -> glueSchemaRegistrySerializationFacade.encode(TRANSPORT_NAME, schema, payload)); - assertEquals("JSON data validation against schema failed.", ex.getMessage()); - } - private AWSSerializerInput prepareInput(String schemaDefinition, String schemaName, String dataFormat) {