-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add message translation to GetSpec (#18130)
* Update SpecActivityImpl to build a VersionedStreamFactory * Enable Protocol Version detection from Stream for SPEC * Print log before the action for better debugging * Fix buffer size for protocol detection * Improve detectVersion error handling * extract constan * Rename attribute for clarity
- Loading branch information
Showing
9 changed files
with
284 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
115 changes: 115 additions & 0 deletions
115
...s-worker/src/test/java/io/airbyte/workers/internal/VersionedAirbyteStreamFactoryTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,115 @@ | ||
/* | ||
* Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
*/ | ||
|
||
package io.airbyte.workers.internal; | ||
|
||
import static org.junit.jupiter.api.Assertions.assertEquals; | ||
import static org.mockito.Mockito.spy; | ||
import static org.mockito.Mockito.verify; | ||
|
||
import io.airbyte.commons.protocol.AirbyteMessageMigrator; | ||
import io.airbyte.commons.protocol.AirbyteMessageSerDeProvider; | ||
import io.airbyte.commons.protocol.AirbyteMessageVersionedMigratorFactory; | ||
import io.airbyte.commons.protocol.migrations.AirbyteMessageMigrationV0; | ||
import io.airbyte.commons.protocol.serde.AirbyteMessageV0Deserializer; | ||
import io.airbyte.commons.protocol.serde.AirbyteMessageV0Serializer; | ||
import io.airbyte.commons.version.Version; | ||
import io.airbyte.protocol.models.AirbyteMessage; | ||
import java.io.BufferedReader; | ||
import java.io.InputStreamReader; | ||
import java.io.StringReader; | ||
import java.nio.charset.Charset; | ||
import java.util.List; | ||
import java.util.stream.Stream; | ||
import org.junit.jupiter.api.BeforeEach; | ||
import org.junit.jupiter.api.Test; | ||
import org.junit.platform.commons.util.ClassLoaderUtils; | ||
|
||
class VersionedAirbyteStreamFactoryTest { | ||
|
||
AirbyteMessageSerDeProvider serDeProvider; | ||
AirbyteMessageVersionedMigratorFactory migratorFactory; | ||
|
||
final static Version defaultVersion = new Version("0.2.0"); | ||
|
||
@BeforeEach | ||
void beforeEach() { | ||
serDeProvider = spy(new AirbyteMessageSerDeProvider( | ||
List.of(new AirbyteMessageV0Deserializer()), | ||
List.of(new AirbyteMessageV0Serializer()))); | ||
serDeProvider.initialize(); | ||
final AirbyteMessageMigrator migrator = new AirbyteMessageMigrator( | ||
List.of(new AirbyteMessageMigrationV0())); | ||
migrator.initialize(); | ||
migratorFactory = spy(new AirbyteMessageVersionedMigratorFactory(migrator)); | ||
} | ||
|
||
@Test | ||
void testCreate() { | ||
final Version initialVersion = new Version("0.1.2"); | ||
final VersionedAirbyteStreamFactory<?> streamFactory = new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion); | ||
|
||
final BufferedReader bufferedReader = new BufferedReader(new StringReader("")); | ||
streamFactory.create(bufferedReader); | ||
|
||
verify(serDeProvider).getDeserializer(initialVersion); | ||
verify(migratorFactory).getVersionedMigrator(initialVersion); | ||
} | ||
|
||
@Test | ||
void testCreateWithVersionDetection() { | ||
final Version initialVersion = new Version("0.0.0"); | ||
final VersionedAirbyteStreamFactory<?> streamFactory = new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion) | ||
.withDetectVersion(true); | ||
|
||
final BufferedReader bufferedReader = | ||
getBuffereredReader("version-detection/logs-with-version.jsonl"); | ||
final Stream<AirbyteMessage> stream = streamFactory.create(bufferedReader); | ||
|
||
long messageCount = stream.toList().size(); | ||
verify(serDeProvider).getDeserializer(initialVersion); | ||
verify(serDeProvider).getDeserializer(new Version("0.5.9")); | ||
assertEquals(1, messageCount); | ||
} | ||
|
||
@Test | ||
void testCreateWithVersionDetectionFallback() { | ||
final Version initialVersion = new Version("0.0.6"); | ||
final VersionedAirbyteStreamFactory<?> streamFactory = new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion) | ||
.withDetectVersion(true); | ||
|
||
final BufferedReader bufferedReader = | ||
getBuffereredReader("version-detection/logs-without-version.jsonl"); | ||
final Stream<AirbyteMessage> stream = streamFactory.create(bufferedReader); | ||
|
||
final long messageCount = stream.toList().size(); | ||
verify(serDeProvider).getDeserializer(initialVersion); | ||
verify(serDeProvider).getDeserializer(defaultVersion); | ||
assertEquals(1, messageCount); | ||
} | ||
|
||
@Test | ||
void testCreateWithVersionDetectionWithoutSpecMessage() { | ||
final Version initialVersion = new Version("0.0.1"); | ||
final VersionedAirbyteStreamFactory<?> streamFactory = new VersionedAirbyteStreamFactory<>(serDeProvider, migratorFactory, initialVersion) | ||
.withDetectVersion(true); | ||
|
||
final BufferedReader bufferedReader = | ||
getBuffereredReader("version-detection/logs-without-spec-message.jsonl"); | ||
final Stream<AirbyteMessage> stream = streamFactory.create(bufferedReader); | ||
|
||
final long messageCount = stream.toList().size(); | ||
verify(serDeProvider).getDeserializer(initialVersion); | ||
verify(serDeProvider).getDeserializer(defaultVersion); | ||
assertEquals(2, messageCount); | ||
} | ||
|
||
BufferedReader getBuffereredReader(final String resourceFile) { | ||
return new BufferedReader( | ||
new InputStreamReader( | ||
ClassLoaderUtils.getDefaultClassLoader().getResourceAsStream(resourceFile), | ||
Charset.defaultCharset())); | ||
} | ||
|
||
} |
Oops, something went wrong.