From 310b9d907489ef4e4cac2f9a99d701db883c7db9 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 27 Jun 2022 17:43:47 +0200 Subject: [PATCH 1/2] [11731] Improved "SecondSync" checks in Standard Destination Acceptance tests --- .../DestinationAcceptanceTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 484d93038ba3..269af648ff63 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -93,6 +93,8 @@ public abstract class DestinationAcceptanceTest { private static final String JOB_ID = "0"; private static final int JOB_ATTEMPT = 0; + private static final String DUMMY_CATALOG_NAME = "DummyCatalog"; + private static final Logger LOGGER = LoggerFactory.getLogger(DestinationAcceptanceTest.class); private TestDestinationEnv testEnv; @@ -415,11 +417,27 @@ public void testSecondSync() throws Exception { final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), AirbyteCatalog.class); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); + final List firstSyncMessages = MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.messageFile).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); final JsonNode config = getConfig(); runSyncAndVerifyStateOutput(config, firstSyncMessages, configuredCatalog, false); + // We need to make sure that other streams\tables\files in the same location will not be + // affected\deleted\overridden by our activities during first, second or any future sync. + // So let's create a dummy data that will be checked after all sync. It should remain the same + final AirbyteCatalog dummyCatalog = + Jsons.deserialize(MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), AirbyteCatalog.class); + dummyCatalog.getStreams().get(0).setName(DUMMY_CATALOG_NAME); + final ConfiguredAirbyteCatalog configuredDummyCatalog = CatalogHelpers.toDefaultConfiguredCatalog(dummyCatalog); + // update messages to set new dummy stream name + firstSyncMessages.stream().filter(message -> message.getRecord() != null) + .forEach(message -> message.getRecord().setStream(DUMMY_CATALOG_NAME)); + // sync dummy data + runSyncAndVerifyStateOutput(config, firstSyncMessages, configuredDummyCatalog, false); + // ---- + + // Run second sync final List secondSyncMessages = Lists.newArrayList( new AirbyteMessage() .withType(Type.RECORD) @@ -442,6 +460,10 @@ public void testSecondSync() throws Exception { runSyncAndVerifyStateOutput(config, secondSyncMessages, configuredCatalog, false); final String defaultSchema = getDefaultSchema(config); retrieveRawRecordsAndAssertSameMessages(catalog, secondSyncMessages, defaultSchema); + + // verify that other streams in the same location were not affected. If something fails here, + // then this need to be fixed in connectors logic to override only required streams + retrieveRawRecordsAndAssertSameMessages(dummyCatalog, firstSyncMessages, defaultSchema); } /** From 3514d0f64027b3566fab3bf115e7dc1320752459 Mon Sep 17 00:00:00 2001 From: ievgeniit Date: Mon, 27 Jun 2022 19:26:37 +0200 Subject: [PATCH 2/2] [11731] code-style fix --- .../standardtest/destination/DestinationAcceptanceTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 269af648ff63..86f12d08bfb8 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -435,7 +435,6 @@ public void testSecondSync() throws Exception { .forEach(message -> message.getRecord().setStream(DUMMY_CATALOG_NAME)); // sync dummy data runSyncAndVerifyStateOutput(config, firstSyncMessages, configuredDummyCatalog, false); - // ---- // Run second sync final List secondSyncMessages = Lists.newArrayList(