diff --git a/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java b/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java index 44ba34f57854..2d91aba22b41 100644 --- a/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java +++ b/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java @@ -35,6 +35,8 @@ class SegmentTrackingClientTest { private static final TrackingIdentity IDENTITY = new TrackingIdentity(AIRBYTE_VERSION, UUID.randomUUID(), EMAIL, false, false, true); private static final UUID WORKSPACE_ID = UUID.randomUUID(); private static final Function MOCK_TRACKING_IDENTITY = (workspaceId) -> IDENTITY; + private static final String AIRBYTE_VERSION_KEY = "airbyte_version"; + private static final String JUMP = "jump"; private Analytics analytics; private SegmentTrackingClient segmentTrackingClient; @@ -61,7 +63,7 @@ void testIdentify() { final IdentifyMessage actual = mockBuilder.getValue().build(); final Map expectedTraits = ImmutableMap.builder() .put("anonymized", IDENTITY.isAnonymousDataCollection()) - .put("airbyte_version", AIRBYTE_VERSION.serialize()) + .put(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION.serialize()) .put("deployment_env", DEPLOYMENT.getDeploymentEnv()) .put("deployment_mode", DEPLOYMENT.getDeploymentMode()) .put("deployment_id", DEPLOYMENT.getDeploymentId()) @@ -87,7 +89,7 @@ void testIdentifyWithRole() { final IdentifyMessage actual = mockBuilder.getValue().build(); final Map expectedTraits = ImmutableMap.builder() .put("airbyte_role", "role") - .put("airbyte_version", AIRBYTE_VERSION.serialize()) + .put(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION.serialize()) .put("anonymized", IDENTITY.isAnonymousDataCollection()) .put("deployment_env", DEPLOYMENT.getDeploymentEnv()) .put("deployment_mode", DEPLOYMENT.getDeploymentMode()) @@ -104,13 +106,13 @@ void testIdentifyWithRole() { void testTrack() { final ArgumentCaptor mockBuilder = ArgumentCaptor.forClass(TrackMessage.Builder.class); final ImmutableMap metadata = - ImmutableMap.of("airbyte_version", AIRBYTE_VERSION.serialize(), "user_id", IDENTITY.getCustomerId()); + ImmutableMap.of(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION.serialize(), "user_id", IDENTITY.getCustomerId()); - segmentTrackingClient.track(WORKSPACE_ID, "jump"); + segmentTrackingClient.track(WORKSPACE_ID, JUMP); verify(analytics).enqueue(mockBuilder.capture()); final TrackMessage actual = mockBuilder.getValue().build(); - assertEquals("jump", actual.event()); + assertEquals(JUMP, actual.event()); assertEquals(IDENTITY.getCustomerId().toString(), actual.userId()); assertEquals(metadata, filterTrackedAtProperty(Objects.requireNonNull(actual.properties()))); } @@ -119,16 +121,16 @@ void testTrack() { void testTrackWithMetadata() { final ArgumentCaptor mockBuilder = ArgumentCaptor.forClass(TrackMessage.Builder.class); final ImmutableMap metadata = ImmutableMap.of( - "airbyte_version", AIRBYTE_VERSION.serialize(), + AIRBYTE_VERSION_KEY, AIRBYTE_VERSION.serialize(), "email", EMAIL, "height", "80 meters", "user_id", IDENTITY.getCustomerId()); - segmentTrackingClient.track(WORKSPACE_ID, "jump", metadata); + segmentTrackingClient.track(WORKSPACE_ID, JUMP, metadata); verify(analytics).enqueue(mockBuilder.capture()); final TrackMessage actual = mockBuilder.getValue().build(); - assertEquals("jump", actual.event()); + assertEquals(JUMP, actual.event()); assertEquals(IDENTITY.getCustomerId().toString(), actual.userId()); assertEquals(metadata, filterTrackedAtProperty(Objects.requireNonNull(actual.properties()))); } diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java index 54455dae445d..a6f0317e7f38 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java @@ -59,13 +59,18 @@ class BootloaderAppTest { private PostgreSQLContainer container; private DataSource configsDataSource; private DataSource jobsDataSource; + private static final String DOCKER = "docker"; + private static final String VERSION_0330_ALPHA = "0.33.0-alpha"; + private static final String VERSION_0320_ALPHA = "0.32.0-alpha"; + private static final String VERSION_0321_ALPHA = "0.32.1-alpha"; + private static final String VERSION_0170_ALPHA = "0.17.0-alpha"; @BeforeEach void setup() { container = new PostgreSQLContainer<>("postgres:13-alpine") .withDatabaseName("public") - .withUsername("docker") - .withPassword("docker"); + .withUsername(DOCKER) + .withPassword(DOCKER); container.start(); configsDataSource = @@ -86,8 +91,6 @@ void cleanup() throws Exception { @Test void testBootloaderAppBlankDb() throws Exception { - val version = "0.33.0-alpha"; - val mockedConfigs = mock(Configs.class); when(mockedConfigs.getConfigDatabaseUrl()).thenReturn(container.getJdbcUrl()); when(mockedConfigs.getConfigDatabaseUser()).thenReturn(container.getUsername()); @@ -95,7 +98,7 @@ void testBootloaderAppBlankDb() throws Exception { when(mockedConfigs.getDatabaseUrl()).thenReturn(container.getJdbcUrl()); when(mockedConfigs.getDatabaseUser()).thenReturn(container.getUsername()); when(mockedConfigs.getDatabasePassword()).thenReturn(container.getPassword()); - when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(version)); + when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(VERSION_0330_ALPHA)); when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true); when(mockedConfigs.getConfigsDatabaseInitializationTimeoutMs()).thenReturn(60000L); when(mockedConfigs.getJobsDatabaseInitializationTimeoutMs()).thenReturn(60000L); @@ -107,8 +110,8 @@ void testBootloaderAppBlankDb() throws Exception { // Although we are able to inject mocked configs into the Bootloader, a particular migration in the // configs database // requires the env var to be set. Flyway prevents injection, so we dynamically set this instead. - environmentVariables.set("DATABASE_USER", "docker"); - environmentVariables.set("DATABASE_PASSWORD", "docker"); + environmentVariables.set("DATABASE_USER", DOCKER); + environmentVariables.set("DATABASE_PASSWORD", DOCKER); environmentVariables.set("DATABASE_URL", container.getJdbcUrl()); try (val configsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES); @@ -133,7 +136,7 @@ void testBootloaderAppBlankDb() throws Exception { assertEquals("0.39.17.001", configsMigrator.getLatestMigration().getVersion().getVersion()); val jobsPersistence = new DefaultJobPersistence(jobDatabase); - assertEquals(version, jobsPersistence.getVersion().get()); + assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get()); assertNotEquals(Optional.empty(), jobsPersistence.getDeployment().get()); } @@ -141,8 +144,6 @@ void testBootloaderAppBlankDb() throws Exception { @Test void testBootloaderAppRunSecretMigration() throws Exception { - val version = "0.33.0-alpha"; - val mockedConfigs = mock(Configs.class); when(mockedConfigs.getConfigDatabaseUrl()).thenReturn(container.getJdbcUrl()); when(mockedConfigs.getConfigDatabaseUser()).thenReturn(container.getUsername()); @@ -150,7 +151,7 @@ void testBootloaderAppRunSecretMigration() throws Exception { when(mockedConfigs.getDatabaseUrl()).thenReturn(container.getJdbcUrl()); when(mockedConfigs.getDatabaseUser()).thenReturn(container.getUsername()); when(mockedConfigs.getDatabasePassword()).thenReturn(container.getPassword()); - when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(version)); + when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(VERSION_0330_ALPHA)); when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true); when(mockedConfigs.getSecretPersistenceType()).thenReturn(TESTING_CONFIG_DB_TABLE); when(mockedConfigs.getConfigsDatabaseInitializationTimeoutMs()).thenReturn(60000L); @@ -181,8 +182,8 @@ void testBootloaderAppRunSecretMigration() throws Exception { // Although we are able to inject mocked configs into the Bootloader, a particular migration in the // configs database requires the env var to be set. Flyway prevents injection, so we dynamically set // this instead. - environmentVariables.set("DATABASE_USER", "docker"); - environmentVariables.set("DATABASE_PASSWORD", "docker"); + environmentVariables.set("DATABASE_USER", DOCKER); + environmentVariables.set("DATABASE_PASSWORD", DOCKER); environmentVariables.set("DATABASE_URL", container.getJdbcUrl()); // Bootstrap the database for the test @@ -265,29 +266,27 @@ void testBootloaderAppRunSecretMigration() throws Exception { void testIsLegalUpgradePredicate() { // starting from no previous version is always legal. assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion("0.17.1-alpha"))); - assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion("0.32.0-alpha"))); - assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion("0.32.1-alpha"))); + assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion(VERSION_0320_ALPHA))); + assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion(VERSION_0321_ALPHA))); assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion("0.33.1-alpha"))); // starting from a version that is pre-breaking migration cannot go past the breaking migration. - assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.17.0-alpha"), new AirbyteVersion("0.17.1-alpha"))); - assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.17.0-alpha"), new AirbyteVersion("0.18.0-alpha"))); - assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.17.0-alpha"), new AirbyteVersion("0.32.0-alpha"))); - assertFalse(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.17.0-alpha"), new AirbyteVersion("0.32.1-alpha"))); - assertFalse(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.17.0-alpha"), new AirbyteVersion("0.33.0-alpha"))); + assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0170_ALPHA), new AirbyteVersion("0.17.1-alpha"))); + assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0170_ALPHA), new AirbyteVersion("0.18.0-alpha"))); + assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0170_ALPHA), new AirbyteVersion(VERSION_0320_ALPHA))); + assertFalse(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0170_ALPHA), new AirbyteVersion(VERSION_0321_ALPHA))); + assertFalse(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0170_ALPHA), new AirbyteVersion(VERSION_0330_ALPHA))); // any migration starting at the breaking migration or after it can upgrade to anything. - assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.32.0-alpha"), new AirbyteVersion("0.32.1-alpha"))); - assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.32.0-alpha"), new AirbyteVersion("0.33.0-alpha"))); - assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.32.1-alpha"), new AirbyteVersion("0.32.1-alpha"))); - assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.32.1-alpha"), new AirbyteVersion("0.33.0-alpha"))); - assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.33.0-alpha"), new AirbyteVersion("0.33.1-alpha"))); - assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.33.0-alpha"), new AirbyteVersion("0.34.0-alpha"))); + assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0320_ALPHA), new AirbyteVersion(VERSION_0321_ALPHA))); + assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0320_ALPHA), new AirbyteVersion(VERSION_0330_ALPHA))); + assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0321_ALPHA), new AirbyteVersion(VERSION_0321_ALPHA))); + assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0321_ALPHA), new AirbyteVersion(VERSION_0330_ALPHA))); + assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0330_ALPHA), new AirbyteVersion("0.33.1-alpha"))); + assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0330_ALPHA), new AirbyteVersion("0.34.0-alpha"))); } @Test void testPostLoadExecutionExecutes() throws Exception { final var testTriggered = new AtomicBoolean(); - val version = "0.33.0-alpha"; - val mockedConfigs = mock(Configs.class); when(mockedConfigs.getConfigDatabaseUrl()).thenReturn(container.getJdbcUrl()); when(mockedConfigs.getConfigDatabaseUser()).thenReturn(container.getUsername()); @@ -295,7 +294,7 @@ void testPostLoadExecutionExecutes() throws Exception { when(mockedConfigs.getDatabaseUrl()).thenReturn(container.getJdbcUrl()); when(mockedConfigs.getDatabaseUser()).thenReturn(container.getUsername()); when(mockedConfigs.getDatabasePassword()).thenReturn(container.getPassword()); - when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(version)); + when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(VERSION_0330_ALPHA)); when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true); when(mockedConfigs.getConfigsDatabaseInitializationTimeoutMs()).thenReturn(60000L); when(mockedConfigs.getJobsDatabaseInitializationTimeoutMs()).thenReturn(60000L); diff --git a/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java b/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java index 12df91995918..f2a09b3a2379 100644 --- a/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java +++ b/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java @@ -16,10 +16,13 @@ class ClisTest { + private static final String ALPHA = "alpha"; + private static final String BETA = "beta"; + @Test void testCreateOptionGroup() { - final Option optionA = new Option("a", "alpha"); - final Option optionB = new Option("b", "beta"); + final Option optionA = new Option("a", ALPHA); + final Option optionB = new Option("b", BETA); final OptionGroup optionGroupExpected = new OptionGroup(); optionGroupExpected.addOption(optionA); optionGroupExpected.addOption(optionB); @@ -38,10 +41,10 @@ void testParse() { final Option optionA = Option.builder("a").required(true).hasArg(true).build(); final Option optionB = Option.builder("b").required(true).hasArg(true).build(); final Options options = new Options().addOption(optionA).addOption(optionB); - final String[] args = {"-a", "alpha", "-b", "beta"}; + final String[] args = {"-a", ALPHA, "-b", BETA}; final CommandLine parsed = Clis.parse(args, options, new DefaultParser()); - assertEquals("alpha", parsed.getOptions()[0].getValue()); - assertEquals("beta", parsed.getOptions()[1].getValue()); + assertEquals(ALPHA, parsed.getOptions()[0].getValue()); + assertEquals(BETA, parsed.getOptions()[1].getValue()); } @Test @@ -49,7 +52,7 @@ void testParseNonConforming() { final Option optionA = Option.builder("a").required(true).hasArg(true).build(); final Option optionB = Option.builder("b").required(true).hasArg(true).build(); final Options options = new Options().addOption(optionA).addOption(optionB); - final String[] args = {"-a", "alpha", "-b", "beta", "-c", "charlie"}; + final String[] args = {"-a", ALPHA, "-b", BETA, "-c", "charlie"}; assertThrows(IllegalArgumentException.class, () -> Clis.parse(args, options, new DefaultParser())); } @@ -58,7 +61,7 @@ void testParseNonConformingWithSyntax() { final Option optionA = Option.builder("a").required(true).hasArg(true).build(); final Option optionB = Option.builder("b").required(true).hasArg(true).build(); final Options options = new Options().addOption(optionA).addOption(optionB); - final String[] args = {"-a", "alpha", "-b", "beta", "-c", "charlie"}; + final String[] args = {"-a", ALPHA, "-b", BETA, "-c", "charlie"}; assertThrows(IllegalArgumentException.class, () -> Clis.parse(args, options, new DefaultParser(), "search")); } @@ -67,10 +70,10 @@ void testRelaxedParser() { final Option optionA = Option.builder("a").required(true).hasArg(true).build(); final Option optionB = Option.builder("b").required(true).hasArg(true).build(); final Options options = new Options().addOption(optionA).addOption(optionB); - final String[] args = {"-a", "alpha", "-b", "beta", "-c", "charlie"}; + final String[] args = {"-a", ALPHA, "-b", BETA, "-c", "charlie"}; final CommandLine parsed = Clis.parse(args, options, Clis.getRelaxedParser()); - assertEquals("alpha", parsed.getOptions()[0].getValue()); - assertEquals("beta", parsed.getOptions()[1].getValue()); + assertEquals(ALPHA, parsed.getOptions()[0].getValue()); + assertEquals(BETA, parsed.getOptions()[1].getValue()); } } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/io/Archives.java b/airbyte-commons/src/main/java/io/airbyte/commons/io/Archives.java index 0e051214efb9..0e81c1d6adf3 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/io/Archives.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/io/Archives.java @@ -49,8 +49,8 @@ private static void compressFile(final Path file, final Path filename, final Tar public static void extractArchive(final Path archiveFile, final Path destinationFolder) throws IOException { final TarArchiveInputStream archive = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(Files.newInputStream(archiveFile)))); - ArchiveEntry entry; - while ((entry = archive.getNextEntry()) != null) { + ArchiveEntry entry = archive.getNextEntry(); + while (entry != null) { final Path newPath = zipSlipProtect(entry, destinationFolder); if (entry.isDirectory()) { Files.createDirectories(newPath); @@ -63,6 +63,7 @@ public static void extractArchive(final Path archiveFile, final Path destination } Files.copy(archive, newPath, StandardCopyOption.REPLACE_EXISTING); } + entry = archive.getNextEntry(); } } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java b/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java index 044a16ac9d79..945bfbf56366 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java @@ -88,9 +88,10 @@ public static List getTail(final int numLines, final Path path) throws I try (final ReversedLinesFileReader fileReader = new ReversedLinesFileReader(file, Charsets.UTF_8)) { final List lines = new ArrayList<>(); - String line; - while ((line = fileReader.readLine()) != null && lines.size() < numLines) { + String line = fileReader.readLine(); + while (line != null && lines.size() < numLines) { lines.add(line); + line = fileReader.readLine(); } Collections.reverse(lines); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java b/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java index e19d22fff6cc..4f02a654b24e 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java @@ -20,13 +20,14 @@ public class LineGobbler implements VoidCallable { private final static Logger LOGGER = LoggerFactory.getLogger(LineGobbler.class); + private final static String GENERIC = "generic"; public static void gobble(final InputStream is, final Consumer consumer) { - gobble(is, consumer, "generic", MdcScope.DEFAULT_BUILDER); + gobble(is, consumer, GENERIC, MdcScope.DEFAULT_BUILDER); } public static void gobble(final InputStream is, final Consumer consumer, final MdcScope.Builder mdcScopeBuilder) { - gobble(is, consumer, "generic", mdcScopeBuilder); + gobble(is, consumer, GENERIC, mdcScopeBuilder); } public static void gobble(final InputStream is, final Consumer consumer, final String caller, final MdcScope.Builder mdcScopeBuilder) { @@ -47,7 +48,7 @@ public static void gobble(final InputStream is, final Consumer consumer, final Consumer consumer, final ExecutorService executor, final Map mdc) { - this(is, consumer, executor, mdc, "generic", MdcScope.DEFAULT_BUILDER); + this(is, consumer, executor, mdc, GENERIC, MdcScope.DEFAULT_BUILDER); } LineGobbler(final InputStream is, @@ -55,7 +56,7 @@ public static void gobble(final InputStream is, final Consumer consumer, final ExecutorService executor, final Map mdc, final MdcScope.Builder mdcScopeBuilder) { - this(is, consumer, executor, mdc, "generic", mdcScopeBuilder); + this(is, consumer, executor, mdc, GENERIC, mdcScopeBuilder); } LineGobbler(final InputStream is, @@ -76,11 +77,12 @@ public static void gobble(final InputStream is, final Consumer consumer, public void voidCall() { MDC.setContextMap(mdc); try { - String line; - while ((line = is.readLine()) != null) { + String line = is.readLine(); + while (line != null) { try (final var mdcScope = containerLogMdcBuilder.build()) { consumer.accept(line); } + line = is.readLine(); } } catch (final IOException i) { LOGGER.warn("{} gobbler IOException: {}. Typically happens when cancelling a job.", caller, i.getMessage()); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index cdc07bfd2cb5..9ff9158135d4 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -34,7 +34,7 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; -@SuppressWarnings("PMD.AvoidReassigningParameters") +@SuppressWarnings({"PMD.AvoidReassigningParameters", "PMD.AvoidCatchingThrowable"}) public class Jsons { // Object Mapper is thread-safe diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java b/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java index 7cb17330738a..dc40576f7b94 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java @@ -21,25 +21,27 @@ public class MoreResources { - @SuppressWarnings("UnstableApiUsage") + private static final String UNSTABLE_API_USAGE = "UnstableApiUsage"; + + @SuppressWarnings(UNSTABLE_API_USAGE) public static String readResource(final String name) throws IOException { final URL resource = Resources.getResource(name); return Resources.toString(resource, StandardCharsets.UTF_8); } - @SuppressWarnings("UnstableApiUsage") + @SuppressWarnings(UNSTABLE_API_USAGE) public static String readResource(final Class klass, final String name) throws IOException { final String rootedName = !name.startsWith("/") ? String.format("/%s", name) : name; final URL url = Resources.getResource(klass, rootedName); return Resources.toString(url, StandardCharsets.UTF_8); } - @SuppressWarnings("UnstableApiUsage") + @SuppressWarnings(UNSTABLE_API_USAGE) public static File readResourceAsFile(final String name) throws URISyntaxException { return new File(Resources.getResource(name).toURI()); } - @SuppressWarnings("UnstableApiUsage") + @SuppressWarnings(UNSTABLE_API_USAGE) public static byte[] readBytes(final String name) throws IOException { final URL resource = Resources.getResource(name); return Resources.toByteArray(resource); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java b/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java index a8c2183ef192..7ee753443e93 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java @@ -23,6 +23,7 @@ import java.io.Writer; import java.util.Iterator; +@SuppressWarnings("PMD.AvoidBranchingStatementAsLastInLoop") public class Yamls { private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java index f17902f48828..0fa6f920fd69 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java @@ -25,25 +25,28 @@ class IOsTest { + private static final String ABC = "abc"; + private static final String FILE = "file"; + @Test void testReadWrite() throws IOException { final Path path = Files.createTempDirectory("tmp"); - final Path filePath = IOs.writeFile(path, "file", "abc"); + final Path filePath = IOs.writeFile(path, FILE, ABC); - assertEquals(path.resolve("file"), filePath); - assertEquals("abc", IOs.readFile(path, "file")); - assertEquals("abc", IOs.readFile(path.resolve("file"))); + assertEquals(path.resolve(FILE), filePath); + assertEquals(ABC, IOs.readFile(path, FILE)); + assertEquals(ABC, IOs.readFile(path.resolve(FILE))); } @Test void testWriteBytes() throws IOException { final Path path = Files.createTempDirectory("tmp"); - final Path filePath = IOs.writeFile(path.resolve("file"), "abc".getBytes(StandardCharsets.UTF_8)); + final Path filePath = IOs.writeFile(path.resolve(FILE), ABC.getBytes(StandardCharsets.UTF_8)); - assertEquals(path.resolve("file"), filePath); - assertEquals("abc", IOs.readFile(path, "file")); + assertEquals(path.resolve(FILE), filePath); + assertEquals(ABC, IOs.readFile(path, FILE)); } @Test diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java index 9a3b4f9b2040..26a2f2c4f0fc 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java @@ -33,6 +33,7 @@ class JsonPathsTest { private static final String EMPTY_RETURN_QUERY = "$.three"; private static final String REPLACEMENT_STRING = "replaced"; private static final JsonNode REPLACEMENT_JSON = Jsons.deserialize("{ \"replacement\": \"replaced\" }"); + private static final String ONE = "one"; @Test void testGetValues() { @@ -74,7 +75,7 @@ void testIsPathPresent() { void testReplaceAtStringLoud() { assertOriginalObjectNotModified(JSON_NODE, () -> { final JsonNode expected = Jsons.clone(JSON_NODE); - ((ArrayNode) expected.get("one")).set(1, REPLACEMENT_STRING); + ((ArrayNode) expected.get(ONE)).set(1, REPLACEMENT_STRING); final JsonNode actual = JsonPaths.replaceAtStringLoud(JSON_NODE, LIST_ONE_QUERY, REPLACEMENT_STRING); assertEquals(expected, actual); @@ -93,7 +94,7 @@ void testReplaceAtStringLoudEmptyPathThrows() { void testReplaceAtString() { assertOriginalObjectNotModified(JSON_NODE, () -> { final JsonNode expected = Jsons.clone(JSON_NODE); - ((ArrayNode) expected.get("one")).set(1, REPLACEMENT_STRING); + ((ArrayNode) expected.get(ONE)).set(1, REPLACEMENT_STRING); final JsonNode actual = JsonPaths.replaceAtString(JSON_NODE, LIST_ONE_QUERY, REPLACEMENT_STRING); assertEquals(expected, actual); @@ -114,7 +115,7 @@ void testReplaceAtStringEmptyReturnNoOp() { void testReplaceAtJsonNodeLoud() { assertOriginalObjectNotModified(JSON_NODE, () -> { final JsonNode expected = Jsons.clone(JSON_NODE); - ((ArrayNode) expected.get("one")).set(1, REPLACEMENT_JSON); + ((ArrayNode) expected.get(ONE)).set(1, REPLACEMENT_JSON); final JsonNode actual = JsonPaths.replaceAtJsonNodeLoud(JSON_NODE, LIST_ONE_QUERY, REPLACEMENT_JSON); assertEquals(expected, actual); @@ -133,9 +134,9 @@ void testReplaceAtJsonNodeLoudEmptyPathThrows() { void testReplaceAtJsonNodeLoudMultipleReplace() { assertOriginalObjectNotModified(JSON_NODE, () -> { final JsonNode expected = Jsons.clone(JSON_NODE); - ((ArrayNode) expected.get("one")).set(0, REPLACEMENT_JSON); - ((ArrayNode) expected.get("one")).set(1, REPLACEMENT_JSON); - ((ArrayNode) expected.get("one")).set(2, REPLACEMENT_JSON); + ((ArrayNode) expected.get(ONE)).set(0, REPLACEMENT_JSON); + ((ArrayNode) expected.get(ONE)).set(1, REPLACEMENT_JSON); + ((ArrayNode) expected.get(ONE)).set(2, REPLACEMENT_JSON); final JsonNode actual = JsonPaths.replaceAtJsonNodeLoud(JSON_NODE, LIST_ALL_QUERY, REPLACEMENT_JSON); assertEquals(expected, actual); @@ -149,7 +150,7 @@ void testReplaceAtJsonNodeLoudMultipleReplace() { @Test void testReplaceAtJsonNodeLoudMultipleReplaceSplatInEmptyArrayThrows() { final JsonNode expected = Jsons.clone(JSON_NODE); - ((ArrayNode) expected.get("one")).removeAll(); + ((ArrayNode) expected.get(ONE)).removeAll(); assertOriginalObjectNotModified(expected, () -> { assertThrows(PathNotFoundException.class, () -> JsonPaths.replaceAtJsonNodeLoud(expected, "$.one[*]", REPLACEMENT_JSON)); @@ -160,7 +161,7 @@ void testReplaceAtJsonNodeLoudMultipleReplaceSplatInEmptyArrayThrows() { void testReplaceAtJsonNode() { assertOriginalObjectNotModified(JSON_NODE, () -> { final JsonNode expected = Jsons.clone(JSON_NODE); - ((ArrayNode) expected.get("one")).set(1, REPLACEMENT_JSON); + ((ArrayNode) expected.get(ONE)).set(1, REPLACEMENT_JSON); final JsonNode actual = JsonPaths.replaceAtJsonNode(JSON_NODE, LIST_ONE_QUERY, REPLACEMENT_JSON); assertEquals(expected, actual); @@ -181,7 +182,7 @@ void testReplaceAtJsonNodeEmptyReturnNoOp() { void testReplaceAt() { assertOriginalObjectNotModified(JSON_NODE, () -> { final JsonNode expected = Jsons.clone(JSON_NODE); - ((ArrayNode) expected.get("one")).set(1, "1-$['one'][1]"); + ((ArrayNode) expected.get(ONE)).set(1, "1-$['one'][1]"); final JsonNode actual = JsonPaths.replaceAt(JSON_NODE, LIST_ONE_QUERY, (node, path) -> Jsons.jsonNode(node + "-" + path)); assertEquals(expected, actual); @@ -192,9 +193,9 @@ void testReplaceAt() { void testReplaceAtMultiple() { assertOriginalObjectNotModified(JSON_NODE, () -> { final JsonNode expected = Jsons.clone(JSON_NODE); - ((ArrayNode) expected.get("one")).set(0, "0-$['one'][0]"); - ((ArrayNode) expected.get("one")).set(1, "1-$['one'][1]"); - ((ArrayNode) expected.get("one")).set(2, "2-$['one'][2]"); + ((ArrayNode) expected.get(ONE)).set(0, "0-$['one'][0]"); + ((ArrayNode) expected.get(ONE)).set(1, "1-$['one'][1]"); + ((ArrayNode) expected.get(ONE)).set(2, "2-$['one'][2]"); final JsonNode actual = JsonPaths.replaceAt(JSON_NODE, LIST_ALL_QUERY, (node, path) -> Jsons.jsonNode(node + "-" + path)); assertEquals(expected, actual); diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java index e4c881b1fb05..15962f2a8505 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java @@ -23,6 +23,14 @@ @SuppressWarnings("PMD.JUnitTestsShouldIncludeAssert") class JsonSchemasTest { + private static final String UNCHECKED = "unchecked"; + private static final String NAME = "name"; + private static final String PROPERTIES = "properties"; + private static final String PETS = "pets"; + private static final String COMPANY = "company"; + private static final String ITEMS = "items"; + private static final String USER = "user"; + @Test void testMutateTypeToArrayStandard() { final JsonNode expectedWithoutType = Jsons.deserialize("{\"test\":\"abc\"}"); @@ -41,7 +49,7 @@ void testMutateTypeToArrayStandard() { assertEquals(expectedWithoutArrayType, actualWithStringType); } - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @Test void testTraverse() throws IOException { final JsonNode jsonWithAllTypes = Jsons.deserialize(MoreResources.readResource("json_schemas/json_with_all_types.json")); @@ -50,23 +58,23 @@ void testTraverse() throws IOException { JsonSchemas.traverseJsonSchema(jsonWithAllTypes, mock); final InOrder inOrder = Mockito.inOrder(mock); inOrder.verify(mock).accept(jsonWithAllTypes, Collections.emptyList()); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("name"), List.of(FieldNameOrList.fieldName("name"))); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("name").get("properties").get("first"), - List.of(FieldNameOrList.fieldName("name"), FieldNameOrList.fieldName("first"))); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("name").get("properties").get("last"), - List.of(FieldNameOrList.fieldName("name"), FieldNameOrList.fieldName("last"))); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("company"), List.of(FieldNameOrList.fieldName("company"))); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets"), List.of(FieldNameOrList.fieldName("pets"))); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets").get("items"), - List.of(FieldNameOrList.fieldName("pets"), FieldNameOrList.list())); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets").get("items").get("properties").get("type"), - List.of(FieldNameOrList.fieldName("pets"), FieldNameOrList.list(), FieldNameOrList.fieldName("type"))); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets").get("items").get("properties").get("number"), - List.of(FieldNameOrList.fieldName("pets"), FieldNameOrList.list(), FieldNameOrList.fieldName("number"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(PROPERTIES).get(NAME), List.of(FieldNameOrList.fieldName(NAME))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(PROPERTIES).get(NAME).get(PROPERTIES).get("first"), + List.of(FieldNameOrList.fieldName(NAME), FieldNameOrList.fieldName("first"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(PROPERTIES).get(NAME).get(PROPERTIES).get("last"), + List.of(FieldNameOrList.fieldName(NAME), FieldNameOrList.fieldName("last"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(PROPERTIES).get(COMPANY), List.of(FieldNameOrList.fieldName(COMPANY))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(PROPERTIES).get(PETS), List.of(FieldNameOrList.fieldName(PETS))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(PROPERTIES).get(PETS).get(ITEMS), + List.of(FieldNameOrList.fieldName(PETS), FieldNameOrList.list())); + inOrder.verify(mock).accept(jsonWithAllTypes.get(PROPERTIES).get(PETS).get(ITEMS).get(PROPERTIES).get("type"), + List.of(FieldNameOrList.fieldName(PETS), FieldNameOrList.list(), FieldNameOrList.fieldName("type"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(PROPERTIES).get(PETS).get(ITEMS).get(PROPERTIES).get("number"), + List.of(FieldNameOrList.fieldName(PETS), FieldNameOrList.list(), FieldNameOrList.fieldName("number"))); inOrder.verifyNoMoreInteractions(); } - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @ValueSource(strings = { "anyOf", "oneOf", @@ -85,18 +93,18 @@ void testTraverseComposite(final String compositeKeyword) throws IOException { inOrder.verify(mock).accept(jsonWithAllTypes, Collections.emptyList()); inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(0), Collections.emptyList()); inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1), Collections.emptyList()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1).get("properties").get("prop1"), + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1).get(PROPERTIES).get("prop1"), List.of(FieldNameOrList.fieldName("prop1"))); inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(2), Collections.emptyList()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(2).get("items"), List.of(FieldNameOrList.list())); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(2).get(ITEMS), List.of(FieldNameOrList.list())); inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(3).get(compositeKeyword).get(0), Collections.emptyList()); inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(3).get(compositeKeyword).get(1), Collections.emptyList()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(3).get(compositeKeyword).get(1).get("items"), + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(3).get(compositeKeyword).get(1).get(ITEMS), List.of(FieldNameOrList.list())); inOrder.verifyNoMoreInteractions(); } - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @Test void testTraverseMultiType() throws IOException { final JsonNode jsonWithAllTypes = Jsons.deserialize(MoreResources.readResource("json_schemas/json_with_array_type_fields.json")); @@ -105,14 +113,14 @@ void testTraverseMultiType() throws IOException { JsonSchemas.traverseJsonSchema(jsonWithAllTypes, mock); final InOrder inOrder = Mockito.inOrder(mock); inOrder.verify(mock).accept(jsonWithAllTypes, Collections.emptyList()); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("company"), List.of(FieldNameOrList.fieldName("company"))); - inOrder.verify(mock).accept(jsonWithAllTypes.get("items"), List.of(FieldNameOrList.list())); - inOrder.verify(mock).accept(jsonWithAllTypes.get("items").get("properties").get("user"), - List.of(FieldNameOrList.list(), FieldNameOrList.fieldName("user"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(PROPERTIES).get(COMPANY), List.of(FieldNameOrList.fieldName(COMPANY))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(ITEMS), List.of(FieldNameOrList.list())); + inOrder.verify(mock).accept(jsonWithAllTypes.get(ITEMS).get(PROPERTIES).get(USER), + List.of(FieldNameOrList.list(), FieldNameOrList.fieldName(USER))); inOrder.verifyNoMoreInteractions(); } - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @Test void testTraverseMultiTypeComposite() throws IOException { final String compositeKeyword = "anyOf"; @@ -123,17 +131,17 @@ void testTraverseMultiTypeComposite() throws IOException { final InOrder inOrder = Mockito.inOrder(mock); inOrder.verify(mock).accept(jsonWithAllTypes, Collections.emptyList()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(0).get("properties").get("company"), - List.of(FieldNameOrList.fieldName("company"))); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1).get("properties").get("organization"), + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(0).get(PROPERTIES).get(COMPANY), + List.of(FieldNameOrList.fieldName(COMPANY))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1).get(PROPERTIES).get("organization"), List.of(FieldNameOrList.fieldName("organization"))); - inOrder.verify(mock).accept(jsonWithAllTypes.get("items"), List.of(FieldNameOrList.list())); - inOrder.verify(mock).accept(jsonWithAllTypes.get("items").get("properties").get("user"), + inOrder.verify(mock).accept(jsonWithAllTypes.get(ITEMS), List.of(FieldNameOrList.list())); + inOrder.verify(mock).accept(jsonWithAllTypes.get(ITEMS).get(PROPERTIES).get(USER), List.of(FieldNameOrList.list(), FieldNameOrList.fieldName("user"))); inOrder.verifyNoMoreInteractions(); } - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @Test void testTraverseArrayTypeWithNoItemsDoNotThrowsException() throws IOException { final JsonNode jsonWithAllTypes = Jsons.deserialize(MoreResources.readResource("json_schemas/json_with_array_type_fields_no_items.json")); diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java index 2c331417058f..d1a844c02a5e 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java @@ -27,50 +27,58 @@ class JsonsTest { + private static final String SERIALIZED_JSON = "{\"str\":\"abc\",\"num\":999,\"numLong\":888}"; + private static final String SERIALIZED_JSON2 = "{\"str\":\"abc\"}"; + private static final String ABC = "abc"; + private static final String DEF = "def"; + private static final String TEST = "test"; + private static final String TEST2 = "test2"; + private static final String XYZ = "xyz"; + @Test void testSerialize() { assertEquals( - "{\"str\":\"abc\",\"num\":999,\"numLong\":888}", - Jsons.serialize(new ToClass("abc", 999, 888L))); + SERIALIZED_JSON, + Jsons.serialize(new ToClass(ABC, 999, 888L))); assertEquals( "{\"test\":\"abc\",\"test2\":\"def\"}", Jsons.serialize( ImmutableMap.of( - "test", "abc", - "test2", "def"))); + TEST, ABC, + TEST2, DEF))); } @Test void testSerializeJsonNode() { assertEquals( - "{\"str\":\"abc\",\"num\":999,\"numLong\":888}", - Jsons.serialize(Jsons.jsonNode(new ToClass("abc", 999, 888L)))); + SERIALIZED_JSON, + Jsons.serialize(Jsons.jsonNode(new ToClass(ABC, 999, 888L)))); assertEquals( "{\"test\":\"abc\",\"test2\":\"def\"}", Jsons.serialize(Jsons.jsonNode(ImmutableMap.of( - "test", "abc", - "test2", "def")))); + TEST, ABC, + TEST2, DEF)))); // issue: 5878 add test for binary node serialization, binary data are serialized into base64 assertEquals( "{\"test\":\"dGVzdA==\"}", Jsons.serialize(Jsons.jsonNode(ImmutableMap.of( - "test", new BinaryNode("test".getBytes(StandardCharsets.UTF_8)))))); + TEST, new BinaryNode("test".getBytes(StandardCharsets.UTF_8)))))); } @Test void testDeserialize() { assertEquals( - new ToClass("abc", 999, 888L), + new ToClass(ABC, 999, 888L), Jsons.deserialize("{\"str\":\"abc\", \"num\": 999, \"numLong\": 888}", ToClass.class)); } @Test void testDeserializeToJsonNode() { assertEquals( - "{\"str\":\"abc\"}", - Jsons.deserialize("{\"str\":\"abc\"}").toString()); + SERIALIZED_JSON2, + Jsons.deserialize(SERIALIZED_JSON2).toString()); assertEquals( "[{\"str\":\"abc\"},{\"str\":\"abc\"}]", @@ -84,19 +92,19 @@ void testDeserializeToJsonNode() { @Test void testTryDeserialize() { assertEquals( - Optional.of(new ToClass("abc", 999, 888L)), + Optional.of(new ToClass(ABC, 999, 888L)), Jsons.tryDeserialize("{\"str\":\"abc\", \"num\": 999, \"numLong\": 888}", ToClass.class)); assertEquals( - Optional.of(new ToClass("abc", 999, 0L)), + Optional.of(new ToClass(ABC, 999, 0L)), Jsons.tryDeserialize("{\"str\":\"abc\", \"num\": 999, \"test\": 888}", ToClass.class)); } @Test void testTryDeserializeToJsonNode() { assertEquals( - Optional.of(Jsons.deserialize("{\"str\":\"abc\"}")), - Jsons.tryDeserialize("{\"str\":\"abc\"}")); + Optional.of(Jsons.deserialize(SERIALIZED_JSON2)), + Jsons.tryDeserialize(SERIALIZED_JSON2)); assertEquals( Optional.empty(), @@ -106,28 +114,28 @@ void testTryDeserializeToJsonNode() { @Test void testToJsonNode() { assertEquals( - "{\"str\":\"abc\",\"num\":999,\"numLong\":888}", - Jsons.jsonNode(new ToClass("abc", 999, 888L)).toString()); + SERIALIZED_JSON, + Jsons.jsonNode(new ToClass(ABC, 999, 888L)).toString()); assertEquals( "{\"test\":\"abc\",\"test2\":\"def\"}", Jsons.jsonNode( ImmutableMap.of( - "test", "abc", - "test2", "def")) + TEST, ABC, + TEST2, DEF)) .toString()); assertEquals( "{\"test\":\"abc\",\"test2\":{\"inner\":1}}", Jsons.jsonNode( ImmutableMap.of( - "test", "abc", - "test2", ImmutableMap.of("inner", 1))) + TEST, ABC, + TEST2, ImmutableMap.of("inner", 1))) .toString()); assertEquals( - Jsons.jsonNode(new ToClass("abc", 999, 888L)), - Jsons.jsonNode(Jsons.jsonNode(new ToClass("abc", 999, 888L)))); + Jsons.jsonNode(new ToClass(ABC, 999, 888L)), + Jsons.jsonNode(Jsons.jsonNode(new ToClass(ABC, 999, 888L)))); } @Test @@ -142,7 +150,7 @@ void testArrayNode() { @Test void testToObject() { - final ToClass expected = new ToClass("abc", 999, 888L); + final ToClass expected = new ToClass(ABC, 999, 888L); assertEquals( expected, Jsons.object(Jsons.jsonNode(expected), ToClass.class)); @@ -158,14 +166,14 @@ void testToObject() { @Test void testTryToObject() { - final ToClass expected = new ToClass("abc", 999, 888L); + final ToClass expected = new ToClass(ABC, 999, 888L); assertEquals( Optional.of(expected), - Jsons.tryObject(Jsons.deserialize("{\"str\":\"abc\",\"num\":999,\"numLong\":888}"), ToClass.class)); + Jsons.tryObject(Jsons.deserialize(SERIALIZED_JSON), ToClass.class)); assertEquals( Optional.of(expected), - Jsons.tryObject(Jsons.deserialize("{\"str\":\"abc\",\"num\":999,\"numLong\":888}"), new TypeReference() {})); + Jsons.tryObject(Jsons.deserialize(SERIALIZED_JSON), new TypeReference() {})); final ToClass emptyExpected = new ToClass(); assertEquals( @@ -195,24 +203,24 @@ void testToBytes() { @Test void testKeys() { // test object json node - final JsonNode jsonNode = Jsons.jsonNode(ImmutableMap.of("test", "abc", "test2", "def")); - assertEquals(Sets.newHashSet("test", "test2"), Jsons.keys(jsonNode)); + final JsonNode jsonNode = Jsons.jsonNode(ImmutableMap.of(TEST, ABC, TEST2, DEF)); + assertEquals(Sets.newHashSet(TEST, TEST2), Jsons.keys(jsonNode)); // test literal jsonNode assertEquals(Collections.emptySet(), Jsons.keys(jsonNode.get("test"))); // test nested object json node. should only return top-level keys. - final JsonNode nestedJsonNode = Jsons.jsonNode(ImmutableMap.of("test", "abc", "test2", ImmutableMap.of("test3", "def"))); - assertEquals(Sets.newHashSet("test", "test2"), Jsons.keys(nestedJsonNode)); + final JsonNode nestedJsonNode = Jsons.jsonNode(ImmutableMap.of(TEST, ABC, TEST2, ImmutableMap.of("test3", "def"))); + assertEquals(Sets.newHashSet(TEST, TEST2), Jsons.keys(nestedJsonNode)); // test array json node - final JsonNode arrayJsonNode = Jsons.jsonNode(ImmutableList.of(ImmutableMap.of("test", "abc", "test2", "def"))); + final JsonNode arrayJsonNode = Jsons.jsonNode(ImmutableList.of(ImmutableMap.of(TEST, ABC, TEST2, DEF))); assertEquals(Collections.emptySet(), Jsons.keys(arrayJsonNode)); } @Test void testToPrettyString() { - final JsonNode jsonNode = Jsons.jsonNode(ImmutableMap.of("test", "abc")); + final JsonNode jsonNode = Jsons.jsonNode(ImmutableMap.of(TEST, ABC)); final String expectedOutput = "" + "{\n" + " \"test\": \"abc\"\n" @@ -228,22 +236,22 @@ void testGetOptional() { assertEquals(Optional.of(Jsons.emptyObject()), Jsons.getOptional(json, "jkl")); assertEquals(Optional.of(Jsons.jsonNode("pqr")), Jsons.getOptional(json, "mno")); assertEquals(Optional.of(Jsons.jsonNode(null)), Jsons.getOptional(json, "stu")); - assertEquals(Optional.empty(), Jsons.getOptional(json, "xyz")); - assertEquals(Optional.empty(), Jsons.getOptional(json, "abc", "xyz")); - assertEquals(Optional.empty(), Jsons.getOptional(json, "abc", "def", "xyz")); - assertEquals(Optional.empty(), Jsons.getOptional(json, "abc", "jkl", "xyz")); - assertEquals(Optional.empty(), Jsons.getOptional(json, "stu", "xyz")); + assertEquals(Optional.empty(), Jsons.getOptional(json, XYZ)); + assertEquals(Optional.empty(), Jsons.getOptional(json, ABC, XYZ)); + assertEquals(Optional.empty(), Jsons.getOptional(json, ABC, DEF, XYZ)); + assertEquals(Optional.empty(), Jsons.getOptional(json, ABC, "jkl", XYZ)); + assertEquals(Optional.empty(), Jsons.getOptional(json, "stu", XYZ)); } @Test void testGetStringOrNull() { final JsonNode json = Jsons.deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": \"mno\", \"pqr\": 1 }"); - assertEquals("ghi", Jsons.getStringOrNull(json, "abc", "def")); + assertEquals("ghi", Jsons.getStringOrNull(json, ABC, DEF)); assertEquals("mno", Jsons.getStringOrNull(json, "jkl")); assertEquals("1", Jsons.getStringOrNull(json, "pqr")); - assertNull(Jsons.getStringOrNull(json, "abc", "def", "xyz")); - assertNull(Jsons.getStringOrNull(json, "xyz")); + assertNull(Jsons.getStringOrNull(json, ABC, DEF, XYZ)); + assertNull(Jsons.getStringOrNull(json, XYZ)); } @Test diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java index 3879bfd37879..bf55754e37c4 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java @@ -23,6 +23,7 @@ class Log4j2ConfigTest { private static final Path TEST_ROOT = Path.of("/tmp/airbyte_tests"); + private static final String LOG_FILENAME = "logs.log"; private Path root; @BeforeEach @@ -35,12 +36,10 @@ void setUp() throws IOException { void testWorkerDispatch() throws InterruptedException { final Logger logger = LoggerFactory.getLogger("testWorkerDispatch"); - final String filename = "logs.log"; - final ExecutorService executor = Executors.newFixedThreadPool(1); executor.submit(() -> { MDC.put("context", "worker"); - MDC.put("job_log_path", root + "/" + filename); + MDC.put("job_log_path", root + "/" + LOG_FILENAME); logger.error("random message testWorkerDispatch"); MDC.clear(); }); @@ -48,41 +47,38 @@ void testWorkerDispatch() throws InterruptedException { executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); - assertTrue(IOs.readFile(root, filename).contains("random message testWorkerDispatch")); + assertTrue(IOs.readFile(root, LOG_FILENAME).contains("random message testWorkerDispatch")); } @Test void testLogSeparateFiles() throws InterruptedException { final Logger logger = LoggerFactory.getLogger("testLogSeparateFiles"); - final String filename = "logs.log"; final Path root1 = root.resolve("1"); final Path root2 = root.resolve("2"); final ExecutorService executor = Executors.newFixedThreadPool(2); executor.submit(() -> { - MDC.put("job_log_path", root1 + "/" + filename); + MDC.put("job_log_path", root1 + "/" + LOG_FILENAME); logger.error("random message 1"); }); executor.submit(() -> { - MDC.put("job_log_path", root2 + "/" + filename); + MDC.put("job_log_path", root2 + "/" + LOG_FILENAME); logger.error("random message 2"); }); executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); - assertTrue(IOs.readFile(root1, filename).contains("random message 1")); - assertTrue(IOs.readFile(root2, filename).contains("random message 2")); + assertTrue(IOs.readFile(root1, LOG_FILENAME).contains("random message 1")); + assertTrue(IOs.readFile(root2, LOG_FILENAME).contains("random message 2")); } @Test void testLogNoJobRoot() throws InterruptedException { final Logger logger = LoggerFactory.getLogger("testWorkerDispatch"); - final String filename = "logs.log"; - final ExecutorService executor = Executors.newFixedThreadPool(1); executor.submit(() -> { logger.error("random message testLogNoJobRoot"); @@ -92,15 +88,13 @@ void testLogNoJobRoot() throws InterruptedException { executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); - assertFalse(Files.exists(root.resolve(filename))); + assertFalse(Files.exists(root.resolve(LOG_FILENAME))); } @Test void testAppDispatch() throws InterruptedException { final Logger logger = LoggerFactory.getLogger("testAppDispatch"); - final String filename = "logs.log"; - final ExecutorService executor = Executors.newFixedThreadPool(1); executor.submit(() -> { MDC.put("workspace_app_root", root.toString()); @@ -111,15 +105,13 @@ void testAppDispatch() throws InterruptedException { executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); - assertTrue(IOs.readFile(root, filename).contains("random message testAppDispatch")); + assertTrue(IOs.readFile(root, LOG_FILENAME).contains("random message testAppDispatch")); } @Test void testLogNoAppRoot() throws InterruptedException { final Logger logger = LoggerFactory.getLogger("testAppDispatch"); - final String filename = "logs.log"; - final ExecutorService executor = Executors.newFixedThreadPool(1); executor.submit(() -> { logger.error("random message testLogNoAppRoot"); @@ -129,7 +121,7 @@ void testLogNoAppRoot() throws InterruptedException { executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); - assertFalse(Files.exists(root.resolve(filename))); + assertFalse(Files.exists(root.resolve(LOG_FILENAME))); } } diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java index 8995f308e3af..d2d5b72dd3a0 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java @@ -19,43 +19,47 @@ class MoreResourcesTest { + private static final String CONTENT_1 = "content1\n"; + private static final String CONTENT_2 = "content2\n"; + private static final String RESOURCE_TEST = "resource_test"; + @Test void testResourceRead() throws IOException { - assertEquals("content1\n", MoreResources.readResource("resource_test")); - assertEquals("content2\n", MoreResources.readResource("subdir/resource_test_sub")); + assertEquals(CONTENT_1, MoreResources.readResource(RESOURCE_TEST)); + assertEquals(CONTENT_2, MoreResources.readResource("subdir/resource_test_sub")); assertThrows(IllegalArgumentException.class, () -> MoreResources.readResource("invalid")); } @Test void testResourceReadWithClass() throws IOException { - assertEquals("content1\n", MoreResources.readResource(MoreResourcesTest.class, "resource_test")); - assertEquals("content2\n", MoreResources.readResource(MoreResourcesTest.class, "subdir/resource_test_sub")); + assertEquals(CONTENT_1, MoreResources.readResource(MoreResourcesTest.class, RESOURCE_TEST)); + assertEquals(CONTENT_2, MoreResources.readResource(MoreResourcesTest.class, "subdir/resource_test_sub")); - assertEquals("content1\n", MoreResources.readResource(MoreResourcesTest.class, "/resource_test")); - assertEquals("content2\n", MoreResources.readResource(MoreResourcesTest.class, "/subdir/resource_test_sub")); + assertEquals(CONTENT_1, MoreResources.readResource(MoreResourcesTest.class, "/resource_test")); + assertEquals(CONTENT_2, MoreResources.readResource(MoreResourcesTest.class, "/subdir/resource_test_sub")); assertThrows(IllegalArgumentException.class, () -> MoreResources.readResource(MoreResourcesTest.class, "invalid")); } @Test void testReadResourceAsFile() throws URISyntaxException { - final File file = MoreResources.readResourceAsFile("resource_test"); - assertEquals("content1\n", IOs.readFile(file.toPath())); + final File file = MoreResources.readResourceAsFile(RESOURCE_TEST); + assertEquals(CONTENT_1, IOs.readFile(file.toPath())); } @Test void testReadBytes() throws IOException { - assertEquals("content1\n", new String(MoreResources.readBytes("resource_test"), StandardCharsets.UTF_8)); - assertEquals("content2\n", new String(MoreResources.readBytes("subdir/resource_test_sub"), StandardCharsets.UTF_8)); + assertEquals(CONTENT_1, new String(MoreResources.readBytes(RESOURCE_TEST), StandardCharsets.UTF_8)); + assertEquals(CONTENT_2, new String(MoreResources.readBytes("subdir/resource_test_sub"), StandardCharsets.UTF_8)); assertThrows(IllegalArgumentException.class, () -> MoreResources.readBytes("invalid")); } @Test void testResourceReadDuplicateName() throws IOException { - assertEquals("content1\n", MoreResources.readResource("resource_test_a")); - assertEquals("content2\n", MoreResources.readResource("subdir/resource_test_a")); + assertEquals(CONTENT_1, MoreResources.readResource("resource_test_a")); + assertEquals(CONTENT_2, MoreResources.readResource("subdir/resource_test_a")); } @Test diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java index 5c2a49e1291e..d2a4e26125ad 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java @@ -13,9 +13,20 @@ class AirbyteVersionTest { + private static final String VERSION_678 = "6.7.8"; + private static final String VERSION_678_OMEGA = "6.7.8-omega"; + private static final String VERSION_678_ALPHA = "6.7.8-alpha"; + private static final String VERSION_678_GAMMA = "6.7.8-gamma"; + private static final String VERSION_679_ALPHA = "6.7.9-alpha"; + private static final String VERSION_680_ALPHA = "6.8.0-alpha"; + private static final String VERSION_6110_ALPHA = "6.11.0-alpha"; + private static final String VERSION_123_PROD = "1.2.3-prod"; + private static final String DEV = "dev"; + private static final String VERSION_380_ALPHA = "3.8.0-alpha"; + @Test void testParseVersion() { - final AirbyteVersion version = new AirbyteVersion("6.7.8"); + final AirbyteVersion version = new AirbyteVersion(VERSION_678); assertEquals("6", version.getMajorVersion()); assertEquals("7", version.getMinorVersion()); assertEquals("8", version.getPatchVersion()); @@ -23,7 +34,7 @@ void testParseVersion() { @Test void testParseVersionWithLabel() { - final AirbyteVersion version = new AirbyteVersion("6.7.8-omega"); + final AirbyteVersion version = new AirbyteVersion(VERSION_678_OMEGA); assertEquals("6", version.getMajorVersion()); assertEquals("7", version.getMinorVersion()); assertEquals("8", version.getPatchVersion()); @@ -31,53 +42,53 @@ void testParseVersionWithLabel() { @Test void testCompatibleVersionCompareTo() { - assertEquals(0, new AirbyteVersion("6.7.8-omega").compatibleVersionCompareTo(new AirbyteVersion("6.7.8-gamma"))); - assertEquals(0, new AirbyteVersion("6.7.8-alpha").compatibleVersionCompareTo(new AirbyteVersion("6.7.9-alpha"))); - assertTrue(0 < new AirbyteVersion("6.8.0-alpha").compatibleVersionCompareTo(new AirbyteVersion("6.7.8-alpha"))); - assertTrue(0 < new AirbyteVersion("11.8.0-alpha").compatibleVersionCompareTo(new AirbyteVersion("6.7.8-alpha"))); - assertTrue(0 < new AirbyteVersion("6.11.0-alpha").compatibleVersionCompareTo(new AirbyteVersion("6.7.8-alpha"))); - assertTrue(0 > new AirbyteVersion("0.8.0-alpha").compatibleVersionCompareTo(new AirbyteVersion("6.7.8-alpha"))); - assertEquals(0, new AirbyteVersion("1.2.3-prod").compatibleVersionCompareTo(new AirbyteVersion("dev"))); - assertEquals(0, new AirbyteVersion("dev").compatibleVersionCompareTo(new AirbyteVersion("1.2.3-prod"))); + assertEquals(0, new AirbyteVersion(VERSION_678_OMEGA).compatibleVersionCompareTo(new AirbyteVersion(VERSION_678_GAMMA))); + assertEquals(0, new AirbyteVersion(VERSION_678_ALPHA).compatibleVersionCompareTo(new AirbyteVersion(VERSION_679_ALPHA))); + assertTrue(0 < new AirbyteVersion(VERSION_680_ALPHA).compatibleVersionCompareTo(new AirbyteVersion(VERSION_678_ALPHA))); + assertTrue(0 < new AirbyteVersion("11.8.0-alpha").compatibleVersionCompareTo(new AirbyteVersion(VERSION_678_ALPHA))); + assertTrue(0 < new AirbyteVersion(VERSION_6110_ALPHA).compatibleVersionCompareTo(new AirbyteVersion(VERSION_678_ALPHA))); + assertTrue(0 > new AirbyteVersion("0.8.0-alpha").compatibleVersionCompareTo(new AirbyteVersion(VERSION_678_ALPHA))); + assertEquals(0, new AirbyteVersion(VERSION_123_PROD).compatibleVersionCompareTo(new AirbyteVersion(DEV))); + assertEquals(0, new AirbyteVersion(DEV).compatibleVersionCompareTo(new AirbyteVersion(VERSION_123_PROD))); } @Test void testPatchVersionCompareTo() { - assertEquals(0, new AirbyteVersion("6.7.8-omega").patchVersionCompareTo(new AirbyteVersion("6.7.8-gamma"))); - assertTrue(0 > new AirbyteVersion("6.7.8-alpha").patchVersionCompareTo(new AirbyteVersion("6.7.9-alpha"))); - assertTrue(0 > new AirbyteVersion("6.7.8-alpha").patchVersionCompareTo(new AirbyteVersion("6.7.11-alpha"))); - assertTrue(0 < new AirbyteVersion("6.8.0-alpha").patchVersionCompareTo(new AirbyteVersion("6.7.8-alpha"))); - assertTrue(0 < new AirbyteVersion("6.11.0-alpha").patchVersionCompareTo(new AirbyteVersion("6.7.8-alpha"))); - assertTrue(0 > new AirbyteVersion("3.8.0-alpha").patchVersionCompareTo(new AirbyteVersion("6.7.8-alpha"))); - assertTrue(0 > new AirbyteVersion("3.8.0-alpha").patchVersionCompareTo(new AirbyteVersion("11.7.8-alpha"))); - assertEquals(0, new AirbyteVersion("1.2.3-prod").patchVersionCompareTo(new AirbyteVersion("dev"))); - assertEquals(0, new AirbyteVersion("dev").patchVersionCompareTo(new AirbyteVersion("1.2.3-prod"))); + assertEquals(0, new AirbyteVersion(VERSION_678_OMEGA).patchVersionCompareTo(new AirbyteVersion(VERSION_678_GAMMA))); + assertTrue(0 > new AirbyteVersion(VERSION_678_ALPHA).patchVersionCompareTo(new AirbyteVersion(VERSION_679_ALPHA))); + assertTrue(0 > new AirbyteVersion(VERSION_678_ALPHA).patchVersionCompareTo(new AirbyteVersion("6.7.11-alpha"))); + assertTrue(0 < new AirbyteVersion(VERSION_680_ALPHA).patchVersionCompareTo(new AirbyteVersion(VERSION_678_ALPHA))); + assertTrue(0 < new AirbyteVersion(VERSION_6110_ALPHA).patchVersionCompareTo(new AirbyteVersion(VERSION_678_ALPHA))); + assertTrue(0 > new AirbyteVersion(VERSION_380_ALPHA).patchVersionCompareTo(new AirbyteVersion(VERSION_678_ALPHA))); + assertTrue(0 > new AirbyteVersion(VERSION_380_ALPHA).patchVersionCompareTo(new AirbyteVersion("11.7.8-alpha"))); + assertEquals(0, new AirbyteVersion(VERSION_123_PROD).patchVersionCompareTo(new AirbyteVersion(DEV))); + assertEquals(0, new AirbyteVersion(DEV).patchVersionCompareTo(new AirbyteVersion(VERSION_123_PROD))); } @Test void testGreaterThan() { - assertFalse(new AirbyteVersion("6.7.8-omega").greaterThan(new AirbyteVersion("6.7.8-gamma"))); - assertFalse(new AirbyteVersion("6.7.8-alpha").greaterThan(new AirbyteVersion("6.7.9-alpha"))); - assertFalse(new AirbyteVersion("6.7.8-alpha").greaterThan(new AirbyteVersion("6.7.11-alpha"))); - assertTrue(new AirbyteVersion("6.8.0-alpha").greaterThan(new AirbyteVersion("6.7.8-alpha"))); - assertTrue(new AirbyteVersion("6.11.0-alpha").greaterThan(new AirbyteVersion("6.7.8-alpha"))); - assertFalse(new AirbyteVersion("3.8.0-alpha").greaterThan(new AirbyteVersion("6.7.8-alpha"))); - assertFalse(new AirbyteVersion("3.8.0-alpha").greaterThan(new AirbyteVersion("11.7.8-alpha"))); - assertFalse(new AirbyteVersion("1.2.3-prod").greaterThan(new AirbyteVersion("dev"))); - assertFalse(new AirbyteVersion("dev").greaterThan(new AirbyteVersion("1.2.3-prod"))); + assertFalse(new AirbyteVersion(VERSION_678_OMEGA).greaterThan(new AirbyteVersion(VERSION_678_GAMMA))); + assertFalse(new AirbyteVersion(VERSION_678_ALPHA).greaterThan(new AirbyteVersion(VERSION_679_ALPHA))); + assertFalse(new AirbyteVersion(VERSION_678_ALPHA).greaterThan(new AirbyteVersion("6.7.11-alpha"))); + assertTrue(new AirbyteVersion(VERSION_680_ALPHA).greaterThan(new AirbyteVersion(VERSION_678_ALPHA))); + assertTrue(new AirbyteVersion(VERSION_6110_ALPHA).greaterThan(new AirbyteVersion(VERSION_678_ALPHA))); + assertFalse(new AirbyteVersion(VERSION_380_ALPHA).greaterThan(new AirbyteVersion(VERSION_678_ALPHA))); + assertFalse(new AirbyteVersion(VERSION_380_ALPHA).greaterThan(new AirbyteVersion("11.7.8-alpha"))); + assertFalse(new AirbyteVersion(VERSION_123_PROD).greaterThan(new AirbyteVersion(DEV))); + assertFalse(new AirbyteVersion(DEV).greaterThan(new AirbyteVersion(VERSION_123_PROD))); } @Test void testLessThan() { - assertFalse(new AirbyteVersion("6.7.8-omega").lessThan(new AirbyteVersion("6.7.8-gamma"))); - assertTrue(new AirbyteVersion("6.7.8-alpha").lessThan(new AirbyteVersion("6.7.9-alpha"))); - assertTrue(new AirbyteVersion("6.7.8-alpha").lessThan(new AirbyteVersion("6.7.11-alpha"))); - assertFalse(new AirbyteVersion("6.8.0-alpha").lessThan(new AirbyteVersion("6.7.8-alpha"))); - assertFalse(new AirbyteVersion("6.11.0-alpha").lessThan(new AirbyteVersion("6.7.8-alpha"))); - assertTrue(new AirbyteVersion("3.8.0-alpha").lessThan(new AirbyteVersion("6.7.8-alpha"))); - assertTrue(new AirbyteVersion("3.8.0-alpha").lessThan(new AirbyteVersion("11.7.8-alpha"))); - assertFalse(new AirbyteVersion("1.2.3-prod").lessThan(new AirbyteVersion("dev"))); - assertFalse(new AirbyteVersion("dev").lessThan(new AirbyteVersion("1.2.3-prod"))); + assertFalse(new AirbyteVersion(VERSION_678_OMEGA).lessThan(new AirbyteVersion(VERSION_678_GAMMA))); + assertTrue(new AirbyteVersion(VERSION_678_ALPHA).lessThan(new AirbyteVersion(VERSION_679_ALPHA))); + assertTrue(new AirbyteVersion(VERSION_678_ALPHA).lessThan(new AirbyteVersion("6.7.11-alpha"))); + assertFalse(new AirbyteVersion(VERSION_680_ALPHA).lessThan(new AirbyteVersion(VERSION_678_ALPHA))); + assertFalse(new AirbyteVersion(VERSION_6110_ALPHA).lessThan(new AirbyteVersion(VERSION_678_ALPHA))); + assertTrue(new AirbyteVersion(VERSION_380_ALPHA).lessThan(new AirbyteVersion(VERSION_678_ALPHA))); + assertTrue(new AirbyteVersion(VERSION_380_ALPHA).lessThan(new AirbyteVersion("11.7.8-alpha"))); + assertFalse(new AirbyteVersion(VERSION_123_PROD).lessThan(new AirbyteVersion(DEV))); + assertFalse(new AirbyteVersion(DEV).lessThan(new AirbyteVersion(VERSION_123_PROD))); } @Test @@ -88,8 +99,7 @@ void testInvalidVersions() { @Test void testSerialize() { - final var devVersion = "dev"; - assertEquals(devVersion, new AirbyteVersion(devVersion).serialize()); + assertEquals(DEV, new AirbyteVersion(DEV).serialize()); final var nonDevVersion = "0.1.2-alpha"; assertEquals(nonDevVersion, new AirbyteVersion(nonDevVersion).serialize()); @@ -103,10 +113,10 @@ void testCheckVersion() { @Test void testCheckOnlyPatchVersion() { - assertFalse(new AirbyteVersion("6.7.8").checkOnlyPatchVersionIsUpdatedComparedTo(new AirbyteVersion("6.7.8"))); + assertFalse(new AirbyteVersion(VERSION_678).checkOnlyPatchVersionIsUpdatedComparedTo(new AirbyteVersion(VERSION_678))); assertFalse(new AirbyteVersion("6.9.8").checkOnlyPatchVersionIsUpdatedComparedTo(new AirbyteVersion("6.8.9"))); assertFalse(new AirbyteVersion("7.7.8").checkOnlyPatchVersionIsUpdatedComparedTo(new AirbyteVersion("6.7.11"))); - assertTrue(new AirbyteVersion("6.7.9").checkOnlyPatchVersionIsUpdatedComparedTo(new AirbyteVersion("6.7.8"))); + assertTrue(new AirbyteVersion("6.7.9").checkOnlyPatchVersionIsUpdatedComparedTo(new AirbyteVersion(VERSION_678))); } } diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java index f287c79ab9e7..eaabe3f8c844 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java @@ -28,69 +28,73 @@ class YamlsTest { + private static final String LINE_BREAK = "---\n"; + private static final String STR_ABC = "str: \"abc\"\n"; + private static final String ABC = "abc"; + @Test void testSerialize() { assertEquals( - "---\n" - + "str: \"abc\"\n" + LINE_BREAK + + STR_ABC + "num: 999\n" + "numLong: 888\n", - Yamls.serialize(new ToClass("abc", 999, 888L))); + Yamls.serialize(new ToClass(ABC, 999, 888L))); assertEquals( - "---\n" + LINE_BREAK + "test: \"abc\"\n" + "test2: \"def\"\n", Yamls.serialize( ImmutableMap.of( - "test", "abc", + "test", ABC, "test2", "def"))); } @Test void testSerializeWithoutQuotes() { assertEquals( - "---\n" + LINE_BREAK + "str: abc\n" + "num: 999\n" + "numLong: 888\n", - Yamls.serializeWithoutQuotes(new ToClass("abc", 999, 888L))); + Yamls.serializeWithoutQuotes(new ToClass(ABC, 999, 888L))); assertEquals( - "---\n" + LINE_BREAK + "test: abc\n" + "test2: def\n", Yamls.serializeWithoutQuotes( ImmutableMap.of( - "test", "abc", + "test", ABC, "test2", "def"))); } @Test void testSerializeJsonNode() { assertEquals( - "---\n" - + "str: \"abc\"\n" + LINE_BREAK + + STR_ABC + "num: 999\n" + "numLong: 888\n", - Yamls.serialize(Jsons.jsonNode(new ToClass("abc", 999, 888L)))); + Yamls.serialize(Jsons.jsonNode(new ToClass(ABC, 999, 888L)))); assertEquals( - "---\n" + LINE_BREAK + "test: \"abc\"\n" + "test2: \"def\"\n", Yamls.serialize(Jsons.jsonNode(ImmutableMap.of( - "test", "abc", + "test", ABC, "test2", "def")))); } @Test void testDeserialize() { assertEquals( - new ToClass("abc", 999, 888L), + new ToClass(ABC, 999, 888L), Yamls.deserialize( - "---\n" - + "str: \"abc\"\n" + LINE_BREAK + + STR_ABC + "num: \"999\"\n" + "numLong: \"888\"\n", ToClass.class)); @@ -101,14 +105,14 @@ void testDeserializeToJsonNode() { assertEquals( "{\"str\":\"abc\"}", Yamls.deserialize( - "---\n" - + "str: \"abc\"\n") + LINE_BREAK + + STR_ABC) .toString()); assertEquals( "[{\"str\":\"abc\"},{\"str\":\"abc\"}]", Yamls.deserialize( - "---\n" + LINE_BREAK + "- str: \"abc\"\n" + "- str: \"abc\"\n") .toString()); diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/S3Logs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/S3Logs.java index 753ab1a41829..fa96e91e76ef 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/S3Logs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/S3Logs.java @@ -175,9 +175,10 @@ private static List getCurrFile(final S3Client s3Client, final String s3 final var is = new ByteArrayInputStream(data); final var currentFileLines = new ArrayList(); try (final var reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { - String temp; - while ((temp = reader.readLine()) != null) { + String temp = reader.readLine(); + while (temp != null) { currentFileLines.add(temp); + temp = reader.readLine(); } } return currentFileLines; diff --git a/airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java b/airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java index a45b291568c5..550674ca9cf5 100644 --- a/airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java +++ b/airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java @@ -24,6 +24,7 @@ class EnvConfigsTest { private Map envMap; private EnvConfigs config; + private final static String ABC = "abc"; private final static String DEV = "dev"; private final static String ABCDEF = "abc/def"; private final static String ROOT = "root"; @@ -157,8 +158,8 @@ void testDockerNetwork() { envMap.put(EnvConfigs.DOCKER_NETWORK, null); assertEquals("host", config.getDockerNetwork()); - envMap.put(EnvConfigs.DOCKER_NETWORK, "abc"); - assertEquals("abc", config.getDockerNetwork()); + envMap.put(EnvConfigs.DOCKER_NETWORK, ABC); + assertEquals(ABC, config.getDockerNetwork()); } @Test @@ -166,7 +167,7 @@ void testTrackingStrategy() { envMap.put(EnvConfigs.TRACKING_STRATEGY, null); assertEquals(Configs.TrackingStrategy.LOGGING, config.getTrackingStrategy()); - envMap.put(EnvConfigs.TRACKING_STRATEGY, "abc"); + envMap.put(EnvConfigs.TRACKING_STRATEGY, ABC); assertEquals(Configs.TrackingStrategy.LOGGING, config.getTrackingStrategy()); envMap.put(EnvConfigs.TRACKING_STRATEGY, "logging"); @@ -184,7 +185,7 @@ void testErrorReportingStrategy() { envMap.put(EnvConfigs.JOB_ERROR_REPORTING_STRATEGY, null); assertEquals(JobErrorReportingStrategy.LOGGING, config.getJobErrorReportingStrategy()); - envMap.put(EnvConfigs.JOB_ERROR_REPORTING_STRATEGY, "abc"); + envMap.put(EnvConfigs.JOB_ERROR_REPORTING_STRATEGY, ABC); assertEquals(JobErrorReportingStrategy.LOGGING, config.getJobErrorReportingStrategy()); envMap.put(EnvConfigs.JOB_ERROR_REPORTING_STRATEGY, "logging"); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java index 04d6e9e4f6e1..ef0e34057f4a 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java @@ -50,6 +50,12 @@ class StatePersistenceTest extends BaseDatabaseConfigPersistenceTest { private ConfigRepository configRepository; private StatePersistence statePersistence; private UUID connectionId; + private static final String STATE_ONE = "\"state1\""; + private static final String STATE_TWO = "\"state2\""; + private static final String STATE_WITH_NAMESPACE = "\"state s1.n1\""; + private static final String STREAM_STATE_2 = "\"state s2\""; + private static final String GLOBAL_STATE = "\"my global state\""; + private static final String STATE = "state"; @Test void testReadingNonExistingState() throws IOException { @@ -103,10 +109,10 @@ void testLegacyMigrationToGlobal() throws IOException { .withStreamStates(Arrays.asList( new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1").withNamespace("n2")) - .withStreamState(Jsons.deserialize("\"state1\"")), + .withStreamState(Jsons.deserialize(STATE_ONE)), new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1")) - .withStreamState(Jsons.deserialize("\"state2\"")))))); + .withStreamState(Jsons.deserialize(STATE_TWO)))))); statePersistence.updateOrCreateState(connectionId, newGlobalState); final StateWrapper storedGlobalState = statePersistence.getCurrentState(connectionId).orElseThrow(); assertEquals(newGlobalState, storedGlobalState); @@ -127,12 +133,12 @@ void testLegacyMigrationToStream() throws IOException { .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1").withNamespace("n1")) - .withStreamState(Jsons.deserialize("\"state s1.n1\""))), + .withStreamState(Jsons.deserialize(STATE_WITH_NAMESPACE))), new AirbyteStateMessage() .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s2")) - .withStreamState(Jsons.deserialize("\"state s2\""))))); + .withStreamState(Jsons.deserialize(STREAM_STATE_2))))); statePersistence.updateOrCreateState(connectionId, newStreamState); final StateWrapper storedStreamState = statePersistence.getCurrentState(connectionId).orElseThrow(); assertEquals(newStreamState, storedStreamState); @@ -145,14 +151,14 @@ void testGlobalReadWrite() throws IOException { .withGlobal(new AirbyteStateMessage() .withType(AirbyteStateType.GLOBAL) .withGlobal(new AirbyteGlobalState() - .withSharedState(Jsons.deserialize("\"my global state\"")) + .withSharedState(Jsons.deserialize(GLOBAL_STATE)) .withStreamStates(Arrays.asList( new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1").withNamespace("n2")) - .withStreamState(Jsons.deserialize("\"state1\"")), + .withStreamState(Jsons.deserialize(STATE_ONE)), new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1")) - .withStreamState(Jsons.deserialize("\"state2\"")))))); + .withStreamState(Jsons.deserialize(STATE_TWO)))))); // Initial write/read loop, making sure we read what we wrote statePersistence.updateOrCreateState(connectionId, state0); @@ -189,14 +195,14 @@ void testGlobalPartialReset() throws IOException { .withGlobal(new AirbyteStateMessage() .withType(AirbyteStateType.GLOBAL) .withGlobal(new AirbyteGlobalState() - .withSharedState(Jsons.deserialize("\"my global state\"")) + .withSharedState(Jsons.deserialize(GLOBAL_STATE)) .withStreamStates(Arrays.asList( new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1").withNamespace("n2")) - .withStreamState(Jsons.deserialize("\"state1\"")), + .withStreamState(Jsons.deserialize(STATE_ONE)), new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1")) - .withStreamState(Jsons.deserialize("\"state2\"")))))); + .withStreamState(Jsons.deserialize(STATE_TWO)))))); // Set the initial state statePersistence.updateOrCreateState(connectionId, state0); @@ -207,11 +213,11 @@ void testGlobalPartialReset() throws IOException { .withGlobal(new AirbyteStateMessage() .withType(AirbyteStateType.GLOBAL) .withGlobal(new AirbyteGlobalState() - .withSharedState(Jsons.deserialize("\"my global state\"")) + .withSharedState(Jsons.deserialize(GLOBAL_STATE)) .withStreamStates(Arrays.asList( new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1")) - .withStreamState(Jsons.deserialize("\"state2\"")))))); + .withStreamState(Jsons.deserialize(STATE_TWO)))))); statePersistence.updateOrCreateState(connectionId, incompletePartialReset); final StateWrapper incompletePartialResetResult = statePersistence.getCurrentState(connectionId).orElseThrow(); Assertions.assertEquals(state0, incompletePartialResetResult); @@ -222,11 +228,11 @@ void testGlobalPartialReset() throws IOException { .withGlobal(new AirbyteStateMessage() .withType(AirbyteStateType.GLOBAL) .withGlobal(new AirbyteGlobalState() - .withSharedState(Jsons.deserialize("\"my global state\"")) + .withSharedState(Jsons.deserialize(GLOBAL_STATE)) .withStreamStates(Arrays.asList( new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1").withNamespace("n2")) - .withStreamState(Jsons.deserialize("\"state1\"")), + .withStreamState(Jsons.deserialize(STATE_ONE)), new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1")) .withStreamState(null))))); @@ -248,14 +254,14 @@ void testGlobalFullReset() throws IOException { .withGlobal(new AirbyteStateMessage() .withType(AirbyteStateType.GLOBAL) .withGlobal(new AirbyteGlobalState() - .withSharedState(Jsons.deserialize("\"my global state\"")) + .withSharedState(Jsons.deserialize(GLOBAL_STATE)) .withStreamStates(Arrays.asList( new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1").withNamespace("n2")) - .withStreamState(Jsons.deserialize("\"state1\"")), + .withStreamState(Jsons.deserialize(STATE_ONE)), new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1")) - .withStreamState(Jsons.deserialize("\"state2\"")))))); + .withStreamState(Jsons.deserialize(STATE_TWO)))))); final StateWrapper fullReset = new StateWrapper() .withStateType(StateType.GLOBAL) @@ -284,7 +290,7 @@ void testGlobalStateAllowsEmptyNameAndNamespace() throws IOException { .withGlobal(new AirbyteStateMessage() .withType(AirbyteStateType.GLOBAL) .withGlobal(new AirbyteGlobalState() - .withSharedState(Jsons.deserialize("\"my global state\"")) + .withSharedState(Jsons.deserialize(GLOBAL_STATE)) .withStreamStates(Arrays.asList( new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("")) @@ -307,12 +313,12 @@ void testStreamReadWrite() throws IOException { .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1").withNamespace("n1")) - .withStreamState(Jsons.deserialize("\"state s1.n1\""))), + .withStreamState(Jsons.deserialize(STATE_WITH_NAMESPACE))), new AirbyteStateMessage() .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s2")) - .withStreamState(Jsons.deserialize("\"state s2\""))))); + .withStreamState(Jsons.deserialize(STREAM_STATE_2))))); // Initial write/read loop, making sure we read what we wrote statePersistence.updateOrCreateState(connectionId, state0); @@ -343,12 +349,12 @@ void testStreamPartialUpdates() throws IOException { .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1").withNamespace("n1")) - .withStreamState(Jsons.deserialize("\"state s1.n1\""))), + .withStreamState(Jsons.deserialize(STATE_WITH_NAMESPACE))), new AirbyteStateMessage() .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s2")) - .withStreamState(Jsons.deserialize("\"state s2\""))))); + .withStreamState(Jsons.deserialize(STREAM_STATE_2))))); statePersistence.updateOrCreateState(connectionId, state0); @@ -376,7 +382,7 @@ void testStreamPartialUpdates() throws IOException { .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s2")) - .withStreamState(Jsons.deserialize("\"state s2\""))))), + .withStreamState(Jsons.deserialize(STREAM_STATE_2))))), partialUpdateResult); // Partial Reset @@ -411,12 +417,12 @@ void testStreamFullReset() throws IOException { .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1").withNamespace("n1")) - .withStreamState(Jsons.deserialize("\"state s1.n1\""))), + .withStreamState(Jsons.deserialize(STATE_WITH_NAMESPACE))), new AirbyteStateMessage() .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s2")) - .withStreamState(Jsons.deserialize("\"state s2\""))))); + .withStreamState(Jsons.deserialize(STREAM_STATE_2))))); statePersistence.updateOrCreateState(connectionId, state0); @@ -448,12 +454,12 @@ void testInconsistentTypeUpdates() throws IOException { .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1").withNamespace("n1")) - .withStreamState(Jsons.deserialize("\"state s1.n1\""))), + .withStreamState(Jsons.deserialize(STATE_WITH_NAMESPACE))), new AirbyteStateMessage() .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s2")) - .withStreamState(Jsons.deserialize("\"state s2\""))))); + .withStreamState(Jsons.deserialize(STREAM_STATE_2))))); statePersistence.updateOrCreateState(connectionId, streamState); Assertions.assertThrows(IllegalStateException.class, () -> { @@ -462,7 +468,7 @@ void testInconsistentTypeUpdates() throws IOException { .withGlobal(new AirbyteStateMessage() .withType(AirbyteStateType.GLOBAL) .withGlobal(new AirbyteGlobalState() - .withSharedState(Jsons.deserialize("\"my global state\"")) + .withSharedState(Jsons.deserialize(GLOBAL_STATE)) .withStreamStates(Arrays.asList( new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("")) @@ -475,8 +481,8 @@ void testInconsistentTypeUpdates() throws IOException { // We should be guarded against those cases let's make sure we don't make things worse if we're in // an inconsistent state - dslContext.insertInto(DSL.table("state")) - .columns(DSL.field("id"), DSL.field("connection_id"), DSL.field("type"), DSL.field("state")) + dslContext.insertInto(DSL.table(STATE)) + .columns(DSL.field("id"), DSL.field("connection_id"), DSL.field("type"), DSL.field(STATE)) .values(UUID.randomUUID(), connectionId, io.airbyte.db.instance.configs.jooq.generated.enums.StateType.GLOBAL, JSONB.valueOf("{}")) .execute(); Assertions.assertThrows(IllegalStateException.class, () -> statePersistence.updateOrCreateState(connectionId, streamState)); @@ -511,9 +517,9 @@ void testStatePersistenceLegacyWriteConsistency() throws IOException { // Making sure we still follow the legacy format final List readStates = dslContext - .selectFrom("state") + .selectFrom(STATE) .where(DSL.field("connection_id").eq(connectionId)) - .fetch().map(r -> Jsons.deserialize(r.get(DSL.field("state", JSONB.class)).data(), State.class)) + .fetch().map(r -> Jsons.deserialize(r.get(DSL.field(STATE, JSONB.class)).data(), State.class)) .stream().toList(); Assertions.assertEquals(1, readStates.size()); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java index 1a9a95403760..fb726c0402bd 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java @@ -53,6 +53,7 @@ * future this will need to independently interact with cloud storage. */ @Slf4j +@SuppressWarnings("PMD.AvoidCatchingThrowable") public class ContainerOrchestratorApp { public static final int MAX_SECONDS_TO_WAIT_FOR_FILE_COPY = 60; diff --git a/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/DefaultAsyncStateManagerTest.java b/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/DefaultAsyncStateManagerTest.java index 3a6098f0f111..4ac2c82bea80 100644 --- a/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/DefaultAsyncStateManagerTest.java +++ b/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/DefaultAsyncStateManagerTest.java @@ -19,6 +19,7 @@ class DefaultAsyncStateManagerTest { private static final KubePodInfo KUBE_POD_INFO = new KubePodInfo("default", "pod1"); + private static final String OUTPUT = "some output value"; private DocumentStoreClient documentStore; private AsyncStateManager stateManager; @@ -42,17 +43,17 @@ void testEmptyWrite() { @Test void testContentfulWrite() { - stateManager.write(KUBE_POD_INFO, AsyncKubePodStatus.SUCCEEDED, "some output value"); + stateManager.write(KUBE_POD_INFO, AsyncKubePodStatus.SUCCEEDED, OUTPUT); final var key = getKey(AsyncKubePodStatus.SUCCEEDED); - verify(documentStore, times(1)).write(key, "some output value"); + verify(documentStore, times(1)).write(key, OUTPUT); } @Test void testReadingOutputWhenItExists() { final var key = getKey(AsyncKubePodStatus.SUCCEEDED); - when(documentStore.read(key)).thenReturn(Optional.of("some output value")); - assertEquals("some output value", stateManager.getOutput(KUBE_POD_INFO)); + when(documentStore.read(key)).thenReturn(Optional.of(OUTPUT)); + assertEquals(OUTPUT, stateManager.getOutput(KUBE_POD_INFO)); } @Test diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java index f92e9af788c6..6567006a63b2 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java @@ -9,6 +9,8 @@ public class IncrementalUtils { + private static final String PROPERTIES = "properties"; + public static String getCursorField(final ConfiguredAirbyteStream stream) { if (stream.getCursorField().size() == 0) { throw new IllegalStateException("No cursor field specified for stream attempting to do incremental."); @@ -20,21 +22,21 @@ public static String getCursorField(final ConfiguredAirbyteStream stream) { } public static JsonSchemaPrimitive getCursorType(final ConfiguredAirbyteStream stream, final String cursorField) { - if (stream.getStream().getJsonSchema().get("properties") == null) { + if (stream.getStream().getJsonSchema().get(PROPERTIES) == null) { throw new IllegalStateException(String.format("No properties found in stream: %s.", stream.getStream().getName())); } - if (stream.getStream().getJsonSchema().get("properties").get(cursorField) == null) { + if (stream.getStream().getJsonSchema().get(PROPERTIES).get(cursorField) == null) { throw new IllegalStateException( String.format("Could not find cursor field: %s in schema for stream: %s.", cursorField, stream.getStream().getName())); } - if (stream.getStream().getJsonSchema().get("properties").get(cursorField).get("type") == null) { + if (stream.getStream().getJsonSchema().get(PROPERTIES).get(cursorField).get("type") == null) { throw new IllegalStateException( String.format("Could not find cursor type for field: %s in schema for stream: %s.", cursorField, stream.getStream().getName())); } - return JsonSchemaPrimitive.valueOf(stream.getStream().getJsonSchema().get("properties").get(cursorField).get("type").asText().toUpperCase()); + return JsonSchemaPrimitive.valueOf(stream.getStream().getJsonSchema().get(PROPERTIES).get(cursorField).get("type").asText().toUpperCase()); } // x < 0 mean replace original diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization.java index dee0e4ddcd09..2c69fecf2e9e 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization.java @@ -49,6 +49,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SuppressWarnings("PMD.AvoidDuplicateLiterals") public class V0_32_8_001__AirbyteConfigDatabaseDenormalization extends BaseJavaMigration { private static final Logger LOGGER = LoggerFactory.getLogger(V0_32_8_001__AirbyteConfigDatabaseDenormalization.class); @@ -156,7 +157,7 @@ private static void createAndPopulateWorkspace(final DSLContext ctx) { LOGGER.info("workspace table populated with " + configsWithMetadata.size() + " records"); } - private static void createAndPopulateActorDefinition(DSLContext ctx) { + private static void createAndPopulateActorDefinition(final DSLContext ctx) { final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); final Field dockerRepository = DSL.field("docker_repository", SQLDataType.VARCHAR(256).nullable(false)); @@ -235,7 +236,7 @@ private static void createAndPopulateActorDefinition(DSLContext ctx) { LOGGER.info("actor_definition table populated with " + destinationDefinitionsWithMetadata.size() + " destination definition records"); } - private static void createAndPopulateActor(DSLContext ctx) { + private static void createAndPopulateActor(final DSLContext ctx) { final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); final Field actorDefinitionId = DSL.field("actor_definition_id", SQLDataType.UUID.nullable(false)); @@ -339,7 +340,7 @@ private static void createAndPopulateActor(DSLContext ctx) { } @VisibleForTesting - static boolean workspaceDoesNotExist(UUID workspaceId, DSLContext ctx) { + static boolean workspaceDoesNotExist(final UUID workspaceId, final DSLContext ctx) { final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); return !ctx.fetchExists(select() .from(table("workspace")) @@ -347,7 +348,7 @@ static boolean workspaceDoesNotExist(UUID workspaceId, DSLContext ctx) { } @VisibleForTesting - static boolean actorDefinitionDoesNotExist(UUID definitionId, DSLContext ctx) { + static boolean actorDefinitionDoesNotExist(final UUID definitionId, final DSLContext ctx) { final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); return !ctx.fetchExists(select() .from(table("actor_definition")) @@ -355,7 +356,7 @@ static boolean actorDefinitionDoesNotExist(UUID definitionId, DSLContext ctx) { } @VisibleForTesting - static boolean actorDoesNotExist(UUID actorId, DSLContext ctx) { + static boolean actorDoesNotExist(final UUID actorId, final DSLContext ctx) { final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); return !ctx.fetchExists(select() .from(table("actor")) @@ -363,7 +364,7 @@ static boolean actorDoesNotExist(UUID actorId, DSLContext ctx) { } @VisibleForTesting - static boolean connectionDoesNotExist(UUID connectionId, DSLContext ctx) { + static boolean connectionDoesNotExist(final UUID connectionId, final DSLContext ctx) { final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); return !ctx.fetchExists(select() .from(table("connection")) @@ -371,7 +372,7 @@ static boolean connectionDoesNotExist(UUID connectionId, DSLContext ctx) { } @VisibleForTesting - static boolean operationDoesNotExist(UUID operationId, DSLContext ctx) { + static boolean operationDoesNotExist(final UUID operationId, final DSLContext ctx) { final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); return !ctx.fetchExists(select() .from(table("operation")) @@ -758,7 +759,7 @@ public enum SourceType implements EnumType { private final String literal; - SourceType(String literal) { + SourceType(final String literal) { this.literal = literal; } @@ -793,7 +794,7 @@ public enum NamespaceDefinitionType implements EnumType { private final String literal; - NamespaceDefinitionType(String literal) { + NamespaceDefinitionType(final String literal) { this.literal = literal; } @@ -827,7 +828,7 @@ public enum StatusType implements EnumType { private final String literal; - StatusType(String literal) { + StatusType(final String literal) { this.literal = literal; } @@ -860,7 +861,7 @@ public enum OperatorType implements EnumType { private final String literal; - OperatorType(String literal) { + OperatorType(final String literal) { this.literal = literal; } @@ -893,7 +894,7 @@ public enum ActorType implements EnumType { private final String literal; - ActorType(String literal) { + ActorType(final String literal) { this.literal = literal; } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_1_001__RemoveForeignKeyFromActorOauth.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_1_001__RemoveForeignKeyFromActorOauth.java index 61bc8f8b0c5f..15bb3995535c 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_1_001__RemoveForeignKeyFromActorOauth.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_1_001__RemoveForeignKeyFromActorOauth.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SuppressWarnings("PMD.AvoidDuplicateLiterals") public class V0_35_1_001__RemoveForeignKeyFromActorOauth extends BaseJavaMigration { private static final Logger LOGGER = LoggerFactory.getLogger(V0_35_1_001__RemoveForeignKeyFromActorOauth.class); @@ -47,7 +48,7 @@ public void migrate(final Context context) throws Exception { } @VisibleForTesting - public static void migrate(DSLContext ctx) { + public static void migrate(final DSLContext ctx) { dropForeignKeyConstraintFromActorOauthTable(ctx); populateActorOauthParameter(ctx); } @@ -136,7 +137,7 @@ private static void populateActorOauthParameter(final DSLContext ctx) { LOGGER.info("actor_oauth_parameter table populated with " + destinationOauthParamRecords + " destination oauth params records"); } - static boolean actorOAuthParamExists(UUID oauthParamId, DSLContext ctx) { + static boolean actorOAuthParamExists(final UUID oauthParamId, final DSLContext ctx) { final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); return ctx.fetchExists(select() .from(table("actor_oauth_parameter")) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalog.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalog.java index 6763f41e1153..9ec7d8e7f1d6 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalog.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalog.java @@ -24,6 +24,7 @@ public class V0_35_26_001__PersistDiscoveredCatalog extends BaseJavaMigration { private static final Logger LOGGER = LoggerFactory.getLogger(V0_35_26_001__PersistDiscoveredCatalog.class); + private static final String ACTOR_CATALOG = "actor_catalog"; @Override public void migrate(final Context context) throws Exception { @@ -48,7 +49,7 @@ private static void createActorCatalog(final DSLContext ctx) { final Field catalog = DSL.field("catalog", SQLDataType.JSONB.nullable(false)); final Field catalogHash = DSL.field("catalog_hash", SQLDataType.VARCHAR(32).nullable(false)); final Field createdAt = DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false)); - ctx.createTableIfNotExists("actor_catalog") + ctx.createTableIfNotExists(ACTOR_CATALOG) .columns(id, catalog, catalogHash, @@ -56,7 +57,7 @@ private static void createActorCatalog(final DSLContext ctx) { .constraints(primaryKey(id)) .execute(); LOGGER.info("actor_catalog table created"); - ctx.createIndexIfNotExists("actor_catalog_catalog_hash_id_idx").on("actor_catalog", "catalog_hash").execute(); + ctx.createIndexIfNotExists("actor_catalog_catalog_hash_id_idx").on(ACTOR_CATALOG, "catalog_hash").execute(); } private static void createCatalogFetchEvent(final DSLContext ctx) { @@ -73,7 +74,7 @@ private static void createCatalogFetchEvent(final DSLContext ctx) { configHash, actorVersion) .constraints(primaryKey(id), - foreignKey(actorCatalogId).references("actor_catalog", "id").onDeleteCascade(), + foreignKey(actorCatalogId).references(ACTOR_CATALOG, "id").onDeleteCascade(), foreignKey(actorId).references("actor", "id").onDeleteCascade()) .execute(); LOGGER.info("actor_catalog_fetch_event table created"); @@ -89,7 +90,7 @@ private static void addConnectionTableForeignKey(final DSLContext ctx) { .dropConstraintIfExists("connection_actor_catalog_id_fk"); ctx.alterTable("connection") .add(constraint("connection_actor_catalog_id_fk").foreignKey(sourceCatalogId) - .references("actor_catalog", "id").onDeleteCascade()) + .references(ACTOR_CATALOG, "id").onDeleteCascade()) .execute(); } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_54_001__ChangeDefaultConnectionName.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_54_001__ChangeDefaultConnectionName.java index 5e96f783c9d0..f3777600be35 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_54_001__ChangeDefaultConnectionName.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_35_54_001__ChangeDefaultConnectionName.java @@ -24,12 +24,13 @@ public class V0_35_54_001__ChangeDefaultConnectionName extends BaseJavaMigration { private static final Logger LOGGER = LoggerFactory.getLogger(V0_35_54_001__ChangeDefaultConnectionName.class); + private static final String NAME = "name"; public static void defaultConnectionName(final DSLContext ctx) { LOGGER.info("Updating connection name column"); final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); - final Field name = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); - List connections = getConnections(ctx); + final Field name = DSL.field(NAME, SQLDataType.VARCHAR(256).nullable(false)); + final List connections = getConnections(ctx); for (final Connection connection : connections) { final Actor sourceActor = getActor(connection.getSourceId(), ctx); @@ -45,12 +46,12 @@ public static void defaultConnectionName(final DSLContext ctx) { static List getConnections(final DSLContext ctx) { LOGGER.info("Get connections having name default"); - final Field name = DSL.field("name", SQLDataType.VARCHAR(36).nullable(false)); + final Field name = DSL.field(NAME, SQLDataType.VARCHAR(36).nullable(false)); final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); final Field sourceId = DSL.field("source_id", SQLDataType.UUID.nullable(false)); final Field destinationId = DSL.field("destination_id", SQLDataType.UUID.nullable(false)); - final Field connectionName = DSL.field("name", SQLDataType.VARCHAR(256).nullable(false)); + final Field connectionName = DSL.field(NAME, SQLDataType.VARCHAR(256).nullable(false)); final Result results = ctx.select(asterisk()).from(table("connection")).where(connectionName.eq("default")).fetch(); return results.stream().map(record -> new Connection( @@ -62,7 +63,7 @@ static List getConnections(final DSLContext ctx) { } static Actor getActor(final UUID actorDefinitionId, final DSLContext ctx) { - final Field name = DSL.field("name", SQLDataType.VARCHAR(36).nullable(false)); + final Field name = DSL.field(NAME, SQLDataType.VARCHAR(36).nullable(false)); final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); final Result results = ctx.select(asterisk()).from(table("actor")).where(id.eq(actorDefinitionId)).fetch(); @@ -83,7 +84,7 @@ public static class Actor { private final String name; - public Actor(String name) { + public Actor(final String name) { this.name = name; } @@ -100,7 +101,7 @@ public static class Connection { private final UUID sourceId; private final UUID destinationId; - public Connection(String name, UUID id, UUID sourceId, UUID destinationId) { + public Connection(final String name, final UUID id, final UUID sourceId, final UUID destinationId) { this.name = name; this.connectionId = id; this.sourceId = sourceId; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevHelper.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevHelper.java index 1ea308c3497d..acd02ed9b14a 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevHelper.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevHelper.java @@ -140,11 +140,12 @@ private static Optional getLastMigrationVersion(final FlywayDa @VisibleForTesting static AirbyteVersion getCurrentAirbyteVersion() { try (final BufferedReader reader = new BufferedReader(new FileReader("../../.env", StandardCharsets.UTF_8))) { - String line; - while ((line = reader.readLine()) != null) { + String line = reader.readLine(); + while (line != null) { if (line.startsWith("VERSION")) { return new AirbyteVersion(line.split("=")[1]); } + line = reader.readLine(); } } catch (final FileNotFoundException e) { throw new IllegalStateException("Cannot find the .env file", e); diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001__MigrateFailureReasonEnumValues.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001__MigrateFailureReasonEnumValues.java index 22924f4fffd5..b8b96aad7908 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001__MigrateFailureReasonEnumValues.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001__MigrateFailureReasonEnumValues.java @@ -27,6 +27,7 @@ public class V0_35_40_001__MigrateFailureReasonEnumValues extends BaseJavaMigration { private static final Logger LOGGER = LoggerFactory.getLogger(V0_35_40_001__MigrateFailureReasonEnumValues.class); + private static final String NULL = ""; @VisibleForTesting static String OLD_MANUAL_CANCELLATION = "manualCancellation"; @@ -179,11 +180,11 @@ public String getInternalMessage() { return internalMessage; } - public void setInternalMessage(String internalMessage) { + public void setInternalMessage(final String internalMessage) { this.internalMessage = internalMessage; } - public FailureReasonForMigration withInternalMessage(String internalMessage) { + public FailureReasonForMigration withInternalMessage(final String internalMessage) { this.internalMessage = internalMessage; return this; } @@ -192,11 +193,11 @@ public String getExternalMessage() { return externalMessage; } - public void setExternalMessage(String externalMessage) { + public void setExternalMessage(final String externalMessage) { this.externalMessage = externalMessage; } - public FailureReasonForMigration withExternalMessage(String externalMessage) { + public FailureReasonForMigration withExternalMessage(final String externalMessage) { this.externalMessage = externalMessage; return this; } @@ -205,11 +206,11 @@ public Metadata getMetadata() { return metadata; } - public void setMetadata(Metadata metadata) { + public void setMetadata(final Metadata metadata) { this.metadata = metadata; } - public FailureReasonForMigration withMetadata(Metadata metadata) { + public FailureReasonForMigration withMetadata(final Metadata metadata) { this.metadata = metadata; return this; } @@ -218,11 +219,11 @@ public String getStacktrace() { return stacktrace; } - public void setStacktrace(String stacktrace) { + public void setStacktrace(final String stacktrace) { this.stacktrace = stacktrace; } - public FailureReasonForMigration withStacktrace(String stacktrace) { + public FailureReasonForMigration withStacktrace(final String stacktrace) { this.stacktrace = stacktrace; return this; } @@ -231,11 +232,11 @@ public Boolean getRetryable() { return retryable; } - public void setRetryable(Boolean retryable) { + public void setRetryable(final Boolean retryable) { this.retryable = retryable; } - public FailureReasonForMigration withRetryable(Boolean retryable) { + public FailureReasonForMigration withRetryable(final Boolean retryable) { this.retryable = retryable; return this; } @@ -244,11 +245,11 @@ public Long getTimestamp() { return timestamp; } - public void setTimestamp(Long timestamp) { + public void setTimestamp(final Long timestamp) { this.timestamp = timestamp; } - public FailureReasonForMigration withTimestamp(Long timestamp) { + public FailureReasonForMigration withTimestamp(final Long timestamp) { this.timestamp = timestamp; return this; } @@ -259,35 +260,35 @@ public String toString() { sb.append(FailureReasonForMigration.class.getName()).append('@').append(Integer.toHexString(System.identityHashCode(this))).append('['); sb.append("failureOrigin"); sb.append('='); - sb.append(((this.failureOrigin == null) ? "" : this.failureOrigin)); + sb.append(((this.failureOrigin == null) ? NULL : this.failureOrigin)); sb.append(','); sb.append("failureType"); sb.append('='); - sb.append(((this.failureType == null) ? "" : this.failureType)); + sb.append(((this.failureType == null) ? NULL : this.failureType)); sb.append(','); sb.append("internalMessage"); sb.append('='); - sb.append(((this.internalMessage == null) ? "" : this.internalMessage)); + sb.append(((this.internalMessage == null) ? NULL : this.internalMessage)); sb.append(','); sb.append("externalMessage"); sb.append('='); - sb.append(((this.externalMessage == null) ? "" : this.externalMessage)); + sb.append(((this.externalMessage == null) ? NULL : this.externalMessage)); sb.append(','); sb.append("metadata"); sb.append('='); - sb.append(((this.metadata == null) ? "" : this.metadata)); + sb.append(((this.metadata == null) ? NULL : this.metadata)); sb.append(','); sb.append("stacktrace"); sb.append('='); - sb.append(((this.stacktrace == null) ? "" : this.stacktrace)); + sb.append(((this.stacktrace == null) ? NULL : this.stacktrace)); sb.append(','); sb.append("retryable"); sb.append('='); - sb.append(((this.retryable == null) ? "" : this.retryable)); + sb.append(((this.retryable == null) ? NULL : this.retryable)); sb.append(','); sb.append("timestamp"); sb.append('='); - sb.append(((this.timestamp == null) ? "" : this.timestamp)); + sb.append(((this.timestamp == null) ? NULL : this.timestamp)); sb.append(','); if (sb.charAt((sb.length() - 1)) == ',') { sb.setCharAt((sb.length() - 1), ']'); @@ -355,11 +356,11 @@ public Boolean getPartialSuccess() { return partialSuccess; } - public void setPartialSuccess(Boolean partialSuccess) { + public void setPartialSuccess(final Boolean partialSuccess) { this.partialSuccess = partialSuccess; } - public AttemptFailureSummaryForMigration withPartialSuccess(Boolean partialSuccess) { + public AttemptFailureSummaryForMigration withPartialSuccess(final Boolean partialSuccess) { this.partialSuccess = partialSuccess; return this; } @@ -370,11 +371,11 @@ public String toString() { sb.append(AttemptFailureSummaryForMigration.class.getName()).append('@').append(Integer.toHexString(System.identityHashCode(this))).append('['); sb.append("failures"); sb.append('='); - sb.append(((this.failures == null) ? "" : this.failures)); + sb.append(((this.failures == null) ? NULL : this.failures)); sb.append(','); sb.append("partialSuccess"); sb.append('='); - sb.append(((this.partialSuccess == null) ? "" : this.partialSuccess)); + sb.append(((this.partialSuccess == null) ? NULL : this.partialSuccess)); sb.append(','); if (sb.charAt((sb.length() - 1)) == ',') { sb.setCharAt((sb.length() - 1), ']'); diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java index f25119bf72b0..d4a3e20a2702 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java @@ -57,6 +57,7 @@ public class MongoUtils { private static final String NULL_TYPE = "null"; private static final String AIRBYTE_SUFFIX = "_aibyte_transform"; private static final int DISCOVER_LIMIT = 10000; + private static final String ID = "_id"; public static JsonSchemaType getType(final BsonType dataType) { return switch (dataType) { @@ -248,7 +249,7 @@ private static List getFieldsName(final MongoCollection collec new Document("$limit", DISCOVER_LIMIT), new Document("$project", new Document("arrayofkeyvalue", new Document("$objectToArray", "$" + fieldName))), new Document("$unwind", "$arrayofkeyvalue"), - new Document("$group", new Document("_id", null).append("allkeys", new Document("$addToSet", "$arrayofkeyvalue.k"))))); + new Document("$group", new Document(ID, null).append("allkeys", new Document("$addToSet", "$arrayofkeyvalue.k"))))); if (output.cursor().hasNext()) { return (List) output.cursor().next().get("allkeys"); } else { @@ -260,13 +261,13 @@ private static List getTypes(final MongoCollection collection, final var fieldName = "$" + name; final AggregateIterable output = collection.aggregate(Arrays.asList( new Document("$limit", DISCOVER_LIMIT), - new Document("$project", new Document("_id", 0).append("fieldType", new Document("$type", fieldName))), - new Document("$group", new Document("_id", new Document("fieldType", "$fieldType")) + new Document("$project", new Document(ID, 0).append("fieldType", new Document("$type", fieldName))), + new Document("$group", new Document(ID, new Document("fieldType", "$fieldType")) .append("count", new Document("$sum", 1))))); final var listOfTypes = new ArrayList(); final var cursor = output.cursor(); while (cursor.hasNext()) { - final var type = ((Document) cursor.next().get("_id")).get("fieldType").toString(); + final var type = ((Document) cursor.next().get(ID)).get("fieldType").toString(); if (!MISSING_TYPE.equals(type) && !NULL_TYPE.equals(type)) { listOfTypes.add(type); } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/JsonUtil.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/JsonUtil.java index 7dbb1254222d..73357ba14c64 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/JsonUtil.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/JsonUtil.java @@ -11,13 +11,15 @@ public class JsonUtil { + private static final String ERROR_MESSAGE = "Can't populate the node type : "; + public static void putBooleanValueIntoJson(final ContainerNode node, final boolean value, final String fieldName) { if (node instanceof ArrayNode) { ((ArrayNode) node).add(value); } else if (node instanceof ObjectNode) { ((ObjectNode) node).put(fieldName, value); } else { - throw new RuntimeException("Can't populate the node type : " + node.getClass().getName()); + throw new RuntimeException(ERROR_MESSAGE + node.getClass().getName()); } } @@ -27,7 +29,7 @@ public static void putLongValueIntoJson(final ContainerNode node, final long } else if (node instanceof ObjectNode) { ((ObjectNode) node).put(fieldName, value); } else { - throw new RuntimeException("Can't populate the node type : " + node.getClass().getName()); + throw new RuntimeException(ERROR_MESSAGE + node.getClass().getName()); } } @@ -37,7 +39,7 @@ public static void putDoubleValueIntoJson(final ContainerNode node, final dou } else if (node instanceof ObjectNode) { ((ObjectNode) node).put(fieldName, value); } else { - throw new RuntimeException("Can't populate the node type : " + node.getClass().getName()); + throw new RuntimeException(ERROR_MESSAGE + node.getClass().getName()); } } @@ -47,7 +49,7 @@ public static void putBigDecimalValueIntoJson(final ContainerNode node, final } else if (node instanceof ObjectNode) { ((ObjectNode) node).put(fieldName, value); } else { - throw new RuntimeException("Can't populate the node type : " + node.getClass().getName()); + throw new RuntimeException(ERROR_MESSAGE + node.getClass().getName()); } } @@ -57,7 +59,7 @@ public static void putStringValueIntoJson(final ContainerNode node, final Str } else if (node instanceof ObjectNode) { ((ObjectNode) node).put(fieldName, value); } else { - throw new RuntimeException("Can't populate the node type : " + node.getClass().getName()); + throw new RuntimeException(ERROR_MESSAGE + node.getClass().getName()); } } @@ -67,7 +69,7 @@ public static void putBytesValueIntoJson(final ContainerNode node, final byte } else if (node instanceof ObjectNode) { ((ObjectNode) node).put(fieldName, value); } else { - throw new RuntimeException("Can't populate the node type : " + node.getClass().getName()); + throw new RuntimeException(ERROR_MESSAGE + node.getClass().getName()); } } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/IncrementalUtilsTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/IncrementalUtilsTest.java index 31a1430cdb6b..94ebd4fcbabe 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/IncrementalUtilsTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/IncrementalUtilsTest.java @@ -27,6 +27,7 @@ class IncrementalUtilsTest { STREAM_NAME, null, Field.of("ascending_inventory_uuid", JsonSchemaType.STRING)); + private static final String ABC = "abc"; @Test void testGetCursorField() { @@ -74,13 +75,13 @@ void testGetCursorTypeCursorHasNoType() { @Test void testCompareCursors() { - assertTrue(IncrementalUtils.compareCursors("abc", "def", JsonSchemaPrimitive.STRING) < 0); - Assertions.assertEquals(0, IncrementalUtils.compareCursors("abc", "abc", JsonSchemaPrimitive.STRING)); + assertTrue(IncrementalUtils.compareCursors(ABC, "def", JsonSchemaPrimitive.STRING) < 0); + Assertions.assertEquals(0, IncrementalUtils.compareCursors(ABC, ABC, JsonSchemaPrimitive.STRING)); assertTrue(IncrementalUtils.compareCursors("1", "2", JsonSchemaPrimitive.NUMBER) < 0); assertTrue(IncrementalUtils.compareCursors("5000000000", "5000000001", JsonSchemaPrimitive.NUMBER) < 0); assertTrue(IncrementalUtils.compareCursors("false", "true", JsonSchemaPrimitive.BOOLEAN) < 0); assertTrue(IncrementalUtils.compareCursors(null, "def", JsonSchemaPrimitive.STRING) < 1); - assertTrue(IncrementalUtils.compareCursors("abc", null, JsonSchemaPrimitive.STRING) > 0); + assertTrue(IncrementalUtils.compareCursors(ABC, null, JsonSchemaPrimitive.STRING) > 0); Assertions.assertEquals(0, IncrementalUtils.compareCursors(null, null, JsonSchemaPrimitive.STRING)); assertThrows(IllegalStateException.class, () -> IncrementalUtils.compareCursors("a", "a", JsonSchemaPrimitive.ARRAY)); assertThrows(IllegalStateException.class, () -> IncrementalUtils.compareCursors("a", "a", JsonSchemaPrimitive.OBJECT)); diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/ConfigsDatabaseMigrationCheckTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/ConfigsDatabaseMigrationCheckTest.java index 322523344d3d..0d2e61b30038 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/ConfigsDatabaseMigrationCheckTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/ConfigsDatabaseMigrationCheckTest.java @@ -21,6 +21,7 @@ class ConfigsDatabaseMigrationCheckTest { private static final String CURRENT_VERSION = "1.2.3"; + private static final String VERSION_2 = "2.0.0"; @Test void testMigrationCheck() { @@ -60,7 +61,7 @@ void testMigrationCheckEqualVersion() { @Test void testMigrationCheckTimeout() { - final var minimumVersion = "2.0.0"; + final var minimumVersion = VERSION_2; final var currentVersion = CURRENT_VERSION; final var migrationVersion = MigrationVersion.fromVersion(currentVersion); final var migrationInfo = mock(MigrationInfo.class); @@ -78,7 +79,7 @@ void testMigrationCheckTimeout() { @Test void testMigrationCheckNullDatabaseAvailibilityCheck() { - final var minimumVersion = "2.0.0"; + final var minimumVersion = VERSION_2; final var currentVersion = CURRENT_VERSION; final var migrationVersion = MigrationVersion.fromVersion(currentVersion); final var migrationInfo = mock(MigrationInfo.class); @@ -95,7 +96,7 @@ void testMigrationCheckNullDatabaseAvailibilityCheck() { @Test void testMigrationCheckNullFlyway() { - final var minimumVersion = "2.0.0"; + final var minimumVersion = VERSION_2; final var databaseAvailabilityCheck = mock(ConfigsDatabaseAvailabilityCheck.class); final var check = new ConfigsDatabaseMigrationCheck(databaseAvailabilityCheck, null, minimumVersion, CommonDatabaseCheckTest.TIMEOUT_MS); Assertions.assertThrows(DatabaseCheckException.class, () -> check.check()); @@ -103,7 +104,7 @@ void testMigrationCheckNullFlyway() { @Test void unavailableFlywayMigrationVersion() { - final var minimumVersion = "2.0.0"; + final var minimumVersion = VERSION_2; final var migrationInfoService = mock(MigrationInfoService.class); final var flyway = mock(Flyway.class); final var databaseAvailabilityCheck = mock(ConfigsDatabaseAvailabilityCheck.class); diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/JobsDatabaseMigrationCheckTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/JobsDatabaseMigrationCheckTest.java index 28e13b856585..c839f0bb44e3 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/JobsDatabaseMigrationCheckTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/JobsDatabaseMigrationCheckTest.java @@ -21,6 +21,7 @@ class JobsDatabaseMigrationCheckTest { private static final String CURRENT_VERSION = "1.2.3"; + private static final String VERSION_2 = "2.0.0"; @Test void testMigrationCheck() { @@ -60,7 +61,7 @@ void testMigrationCheckEqualVersion() { @Test void testMigrationCheckTimeout() { - final var minimumVersion = "2.0.0"; + final var minimumVersion = VERSION_2; final var currentVersion = CURRENT_VERSION; final var migrationVersion = MigrationVersion.fromVersion(currentVersion); final var migrationInfo = mock(MigrationInfo.class); @@ -78,7 +79,7 @@ void testMigrationCheckTimeout() { @Test void testMigrationCheckNullDatabaseAvailabilityCheck() { - final var minimumVersion = "2.0.0"; + final var minimumVersion = VERSION_2; final var currentVersion = CURRENT_VERSION; final var migrationVersion = MigrationVersion.fromVersion(currentVersion); final var migrationInfo = mock(MigrationInfo.class); @@ -95,7 +96,7 @@ void testMigrationCheckNullDatabaseAvailabilityCheck() { @Test void testMigrationCheckNullFlyway() { - final var minimumVersion = "2.0.0"; + final var minimumVersion = VERSION_2; final var databaseAvailabilityCheck = mock(JobsDatabaseAvailabilityCheck.class); final var check = new JobsDatabaseMigrationCheck(databaseAvailabilityCheck, null, minimumVersion, CommonDatabaseCheckTest.TIMEOUT_MS); Assertions.assertThrows(DatabaseCheckException.class, () -> check.check()); @@ -103,7 +104,7 @@ void testMigrationCheckNullFlyway() { @Test void unavailableFlywayMigrationVersion() { - final var minimumVersion = "2.0.0"; + final var minimumVersion = VERSION_2; final var migrationInfoService = mock(MigrationInfoService.class); final var flyway = mock(Flyway.class); final var databaseAvailabilityCheck = mock(JobsDatabaseAvailabilityCheck.class); diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java index 8b377d19c0cd..e3c3cf7829bd 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/FlywayFactoryTest.java @@ -18,41 +18,40 @@ */ class FlywayFactoryTest extends CommonFactoryTest { + private static final String INSTALLED_BY = "test"; + private static final String DB_IDENTIFIER = "test"; + @Test void testCreatingAFlywayInstance() { - final String installedBy = "test"; - final String dbIdentifier = "test"; final String baselineVersion = "1.2.3"; final String baselineDescription = "A test baseline description"; final boolean baselineOnMigrate = true; final String migrationFileLocation = "classpath:io/airbyte/db/instance/toys/migrations"; final DataSource dataSource = DatabaseConnectionHelper.createDataSource(container); final Flyway flyway = - FlywayFactory.create(dataSource, installedBy, dbIdentifier, baselineVersion, baselineDescription, baselineOnMigrate, migrationFileLocation); + FlywayFactory.create(dataSource, INSTALLED_BY, DB_IDENTIFIER, baselineVersion, baselineDescription, baselineOnMigrate, migrationFileLocation); assertNotNull(flyway); assertTrue(flyway.getConfiguration().isBaselineOnMigrate()); assertEquals(baselineDescription, flyway.getConfiguration().getBaselineDescription()); assertEquals(baselineVersion, flyway.getConfiguration().getBaselineVersion().getVersion()); assertEquals(baselineOnMigrate, flyway.getConfiguration().isBaselineOnMigrate()); - assertEquals(installedBy, flyway.getConfiguration().getInstalledBy()); - assertEquals(String.format(FlywayFactory.MIGRATION_TABLE_FORMAT, dbIdentifier), flyway.getConfiguration().getTable()); + assertEquals(INSTALLED_BY, flyway.getConfiguration().getInstalledBy()); + assertEquals(String.format(FlywayFactory.MIGRATION_TABLE_FORMAT, DB_IDENTIFIER), flyway.getConfiguration().getTable()); assertEquals(migrationFileLocation, flyway.getConfiguration().getLocations()[0].getDescriptor()); } @Test void testCreatingAFlywayInstanceWithDefaults() { - final String installedBy = "test"; - final String dbIdentifier = "test"; final String migrationFileLocation = "classpath:io/airbyte/db/instance/toys/migrations"; final DataSource dataSource = DatabaseConnectionHelper.createDataSource(container); - final Flyway flyway = FlywayFactory.create(dataSource, installedBy, dbIdentifier, migrationFileLocation); + final Flyway flyway = FlywayFactory.create(dataSource, INSTALLED_BY, DB_IDENTIFIER, migrationFileLocation); assertNotNull(flyway); assertTrue(flyway.getConfiguration().isBaselineOnMigrate()); assertEquals(FlywayFactory.BASELINE_DESCRIPTION, flyway.getConfiguration().getBaselineDescription()); assertEquals(FlywayFactory.BASELINE_VERSION, flyway.getConfiguration().getBaselineVersion().getVersion()); assertEquals(FlywayFactory.BASELINE_ON_MIGRATION, flyway.getConfiguration().isBaselineOnMigrate()); - assertEquals(installedBy, flyway.getConfiguration().getInstalledBy()); - assertEquals(String.format(FlywayFactory.MIGRATION_TABLE_FORMAT, dbIdentifier), flyway.getConfiguration().getTable()); + assertEquals(INSTALLED_BY, flyway.getConfiguration().getInstalledBy()); + assertEquals(String.format(FlywayFactory.MIGRATION_TABLE_FORMAT, DB_IDENTIFIER), flyway.getConfiguration().getTable()); assertEquals(migrationFileLocation, flyway.getConfiguration().getLocations()[0].getDescriptor()); } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/SetupForNormalizedTablesTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/SetupForNormalizedTablesTest.java index 2093ade5d02d..ea2df07c99d9 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/SetupForNormalizedTablesTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/SetupForNormalizedTablesTest.java @@ -89,6 +89,7 @@ public class SetupForNormalizedTablesTest { private static final Field CREATED_AT = field("created_at", OffsetDateTime.class); private static final Field UPDATED_AT = field("updated_at", OffsetDateTime.class); private static final Instant NOW = Instant.parse("2021-12-15T20:30:40.00Z"); + private static final String CONNECTION_SPEC = "'{\"name\":\"John\", \"age\":30, \"car\":null}'"; public static void setup(final DSLContext context) { createConfigInOldTable(context, standardWorkspace(), ConfigSchema.STANDARD_WORKSPACE); @@ -202,7 +203,7 @@ public static List standardSourceDefinitions() { private static ConnectorSpecification connectorSpecification() { return new ConnectorSpecification() .withAuthSpecification(new AuthSpecification().withAuthType(AuthType.OAUTH_2_0)) - .withConnectionSpecification(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withConnectionSpecification(Jsons.jsonNode(CONNECTION_SPEC)) .withDocumentationUrl(URI.create("whatever")) .withAdvancedAuth(null) .withChangelogUrl(URI.create("whatever")) @@ -239,14 +240,14 @@ public static List sourceConnections() { .withTombstone(false) .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) .withWorkspaceId(WORKSPACE_ID) - .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withConfiguration(Jsons.jsonNode(CONNECTION_SPEC)) .withSourceId(SOURCE_ID_1); final SourceConnection sourceConnection2 = new SourceConnection() .withName("source-2") .withTombstone(false) .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) .withWorkspaceId(WORKSPACE_ID) - .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withConfiguration(Jsons.jsonNode(CONNECTION_SPEC)) .withSourceId(SOURCE_ID_2); return Arrays.asList(sourceConnection1, sourceConnection2); } @@ -257,26 +258,26 @@ public static List destinationConnections() { .withTombstone(false) .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) .withWorkspaceId(WORKSPACE_ID) - .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withConfiguration(Jsons.jsonNode(CONNECTION_SPEC)) .withDestinationId(DESTINATION_ID_1); final DestinationConnection destinationConnection2 = new DestinationConnection() .withName("destination-2") .withTombstone(false) .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) .withWorkspaceId(WORKSPACE_ID) - .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withConfiguration(Jsons.jsonNode(CONNECTION_SPEC)) .withDestinationId(DESTINATION_ID_2); return Arrays.asList(destinationConnection1, destinationConnection2); } public static List sourceOauthParameters() { final SourceOAuthParameter sourceOAuthParameter1 = new SourceOAuthParameter() - .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withConfiguration(Jsons.jsonNode(CONNECTION_SPEC)) .withWorkspaceId(null) .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) .withOauthParameterId(SOURCE_OAUTH_PARAMETER_ID_1); final SourceOAuthParameter sourceOAuthParameter2 = new SourceOAuthParameter() - .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withConfiguration(Jsons.jsonNode(CONNECTION_SPEC)) .withWorkspaceId(WORKSPACE_ID) .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) .withOauthParameterId(SOURCE_OAUTH_PARAMETER_ID_2); @@ -285,12 +286,12 @@ public static List sourceOauthParameters() { public static List destinationOauthParameters() { final DestinationOAuthParameter destinationOAuthParameter1 = new DestinationOAuthParameter() - .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withConfiguration(Jsons.jsonNode(CONNECTION_SPEC)) .withWorkspaceId(null) .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) .withOauthParameterId(DESTINATION_OAUTH_PARAMETER_ID_1); final DestinationOAuthParameter destinationOAuthParameter2 = new DestinationOAuthParameter() - .withConfiguration(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'")) + .withConfiguration(Jsons.jsonNode(CONNECTION_SPEC)) .withWorkspaceId(WORKSPACE_ID) .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) .withOauthParameterId(DESTINATION_OAUTH_PARAMETER_ID_2); @@ -408,16 +409,16 @@ private static ConfiguredAirbyteCatalog getConfiguredCatalog() { public static List standardSyncStates() { final StandardSyncState standardSyncState1 = new StandardSyncState() .withConnectionId(CONNECTION_ID_1) - .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + .withState(new State().withState(Jsons.jsonNode(CONNECTION_SPEC))); final StandardSyncState standardSyncState2 = new StandardSyncState() .withConnectionId(CONNECTION_ID_2) - .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + .withState(new State().withState(Jsons.jsonNode(CONNECTION_SPEC))); final StandardSyncState standardSyncState3 = new StandardSyncState() .withConnectionId(CONNECTION_ID_3) - .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + .withState(new State().withState(Jsons.jsonNode(CONNECTION_SPEC))); final StandardSyncState standardSyncState4 = new StandardSyncState() .withConnectionId(CONNECTION_ID_4) - .withState(new State().withState(Jsons.jsonNode("'{\"name\":\"John\", \"age\":30, \"car\":null}'"))); + .withState(new State().withState(Jsons.jsonNode(CONNECTION_SPEC))); return Arrays.asList(standardSyncState1, standardSyncState2, standardSyncState3, standardSyncState4); } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java index 6b4340ce227b..eb8019ee94a9 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test.java @@ -60,6 +60,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +@SuppressWarnings("PMD.AvoidDuplicateLiterals") class V0_32_8_001__AirbyteConfigDatabaseDenormalization_Test extends AbstractConfigsDatabaseTest { @Test diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalogTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalogTest.java index 2bed6b31778e..2a9a98f8ec44 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalogTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_35_26_001__PersistDiscoveredCatalogTest.java @@ -18,6 +18,8 @@ class V0_35_26_001__PersistDiscoveredCatalogTest extends AbstractConfigsDatabaseTest { + private static final String NAME = "name"; + @Test void test() throws SQLException, IOException { final DSLContext context = getDslContext(); @@ -36,7 +38,7 @@ private void assertCanInsertData(final DSLContext ctx) { ctx.insertInto(DSL.table("workspace")) .columns( DSL.field("id"), - DSL.field("name"), + DSL.field(NAME), DSL.field("slug"), DSL.field("initial_setup_complete")) .values( @@ -48,14 +50,14 @@ private void assertCanInsertData(final DSLContext ctx) { ctx.insertInto(DSL.table("actor_definition")) .columns( DSL.field("id"), - DSL.field("name"), + DSL.field(NAME), DSL.field("docker_repository"), DSL.field("docker_image_tag"), DSL.field("actor_type"), DSL.field("spec")) .values( actorDefinitionId, - "name", + NAME, "repo", "1.0.0", ActorType.source, @@ -66,7 +68,7 @@ private void assertCanInsertData(final DSLContext ctx) { DSL.field("id"), DSL.field("workspace_id"), DSL.field("actor_definition_id"), - DSL.field("name"), + DSL.field(NAME), DSL.field("configuration"), DSL.field("actor_type"), DSL.field("created_at"), diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_39_17_001__AddStreamDescriptorsToStateTableTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_39_17_001__AddStreamDescriptorsToStateTableTest.java index 021e546c4853..fbd22f66d519 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_39_17_001__AddStreamDescriptorsToStateTableTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_39_17_001__AddStreamDescriptorsToStateTableTest.java @@ -24,6 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SuppressWarnings("PMD.AvoidDuplicateLiterals") class V0_39_17_001__AddStreamDescriptorsToStateTableTest extends AbstractConfigsDatabaseTest { private static final Logger LOGGER = LoggerFactory.getLogger(V0_39_17_001__AddStreamDescriptorsToStateTableTest.class); diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/development/MigrationDevHelperTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/development/MigrationDevHelperTest.java index 1577bfe98065..1f0f6cf5b9fc 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/development/MigrationDevHelperTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/development/MigrationDevHelperTest.java @@ -14,6 +14,8 @@ @SuppressWarnings({"PMD.AvoidUsingHardCodedIP", "PMD.JUnitTestsShouldIncludeAssert"}) class MigrationDevHelperTest { + private static final String VERSION_0113_ALPHA = "0.11.3-alpha"; + @Test void testGetCurrentAirbyteVersion() { // Test that this method will not throw any exception. @@ -29,7 +31,7 @@ void testGetAirbyteVersion() { @Test void testFormatAirbyteVersion() { - final AirbyteVersion airbyteVersion = new AirbyteVersion("0.11.3-alpha"); + final AirbyteVersion airbyteVersion = new AirbyteVersion(VERSION_0113_ALPHA); assertEquals("0_11_3", MigrationDevHelper.formatAirbyteVersion(airbyteVersion)); } @@ -43,17 +45,17 @@ void testGetMigrationId() { void testGetNextMigrationVersion() { // Migration version does not exist assertEquals("0.11.3.001", MigrationDevHelper.getNextMigrationVersion( - new AirbyteVersion("0.11.3-alpha"), + new AirbyteVersion(VERSION_0113_ALPHA), Optional.empty()).getVersion()); // Airbyte version is greater assertEquals("0.11.3.001", MigrationDevHelper.getNextMigrationVersion( - new AirbyteVersion("0.11.3-alpha"), + new AirbyteVersion(VERSION_0113_ALPHA), Optional.of(MigrationVersion.fromVersion("0.10.9.003"))).getVersion()); // Airbyte version is equal to migration version assertEquals("0.11.3.004", MigrationDevHelper.getNextMigrationVersion( - new AirbyteVersion("0.11.3-alpha"), + new AirbyteVersion(VERSION_0113_ALPHA), Optional.of(MigrationVersion.fromVersion("0.11.3.003"))).getVersion()); // Migration version is greater diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java index 82f1151dab9b..d6a4b139a2bf 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java @@ -45,6 +45,7 @@ class TestJdbcUtils { private String dbName; + private static final String ONE_POINT_0 = "1.0,"; private static final List RECORDS_AS_JSON = Lists.newArrayList( Jsons.jsonNode(ImmutableMap.of("id", 1, "name", "picard")), @@ -261,11 +262,11 @@ private static void insertRecordOfEachType(final Connection connection) throws S + "1," + "1," + "1," - + "1.0," - + "1.0," - + "1.0," + + ONE_POINT_0 + + ONE_POINT_0 + + ONE_POINT_0 + "1," - + "1.0," + + ONE_POINT_0 + "'a'," + "'a'," + "'2020-11-01'," diff --git a/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java b/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java index 224746c6c542..6dcf3c554b29 100644 --- a/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java +++ b/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java @@ -20,6 +20,8 @@ class JsonSchemaValidatorTest { + private static final String PROPERTIES = "properties"; + private static final JsonNode VALID_SCHEMA = Jsons.deserialize( "{\n" + " \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n" + @@ -91,11 +93,11 @@ void test() throws IOException { final File schemaFile = IOs.writeFile(Files.createTempDirectory("test"), "schema.json", schema).toFile(); // outer object - assertTrue(JsonSchemaValidator.getSchema(schemaFile).get("properties").has("field1")); - assertFalse(JsonSchemaValidator.getSchema(schemaFile).get("properties").has("field2")); + assertTrue(JsonSchemaValidator.getSchema(schemaFile).get(PROPERTIES).has("field1")); + assertFalse(JsonSchemaValidator.getSchema(schemaFile).get(PROPERTIES).has("field2")); // inner object - assertTrue(JsonSchemaValidator.getSchema(schemaFile, "InnerObject").get("properties").has("field2")); - assertFalse(JsonSchemaValidator.getSchema(schemaFile, "InnerObject").get("properties").has("field1")); + assertTrue(JsonSchemaValidator.getSchema(schemaFile, "InnerObject").get(PROPERTIES).has("field2")); + assertFalse(JsonSchemaValidator.getSchema(schemaFile, "InnerObject").get(PROPERTIES).has("field1")); // non-existent object assertNull(JsonSchemaValidator.getSchema(schemaFile, "NonExistentObject")); } diff --git a/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/MetricsQueriesTest.java b/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/MetricsQueriesTest.java index c52596168640..73c79eda185c 100644 --- a/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/MetricsQueriesTest.java +++ b/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/MetricsQueriesTest.java @@ -49,6 +49,10 @@ public class MetricsQueriesTest { private static final String USER = "user"; private static final String PASS = "hunter2"; + private static final String SRC = "src"; + private static final String DEST = "dst"; + private static final String DISPLAY_NAME = "should not error out or return any result if not applicable"; + private static final String CONN = "conn"; private static final UUID SRC_DEF_ID = UUID.randomUUID(); private static final UUID DST_DEF_ID = UUID.randomUUID(); @@ -104,15 +108,15 @@ void shouldReturnReleaseStages() throws SQLException { // create src and dst configDb.transaction( ctx -> ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE) - .values(srcId, UUID.randomUUID(), SRC_DEF_ID, "src", JSONB.valueOf("{}"), ActorType.source) - .values(dstId, UUID.randomUUID(), DST_DEF_ID, "dst", JSONB.valueOf("{}"), ActorType.destination) + .values(srcId, UUID.randomUUID(), SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source) + .values(dstId, UUID.randomUUID(), DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination) .execute()); final var res = configDb.query(ctx -> MetricQueries.srcIdAndDestIdToReleaseStages(ctx, srcId, dstId)); assertEquals(List.of(ReleaseStage.beta, ReleaseStage.generally_available), res); } @Test - @DisplayName("should not error out or return any result if not applicable") + @DisplayName(DISPLAY_NAME) void shouldReturnNothingIfNotApplicable() throws SQLException { final var res = configDb.query(ctx -> MetricQueries.srcIdAndDestIdToReleaseStages(ctx, UUID.randomUUID(), UUID.randomUUID())); assertEquals(0, res.size()); @@ -137,8 +141,8 @@ void shouldReturnReleaseStages() throws SQLException { // create src and dst configDb.transaction( ctx -> ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE) - .values(srcId, UUID.randomUUID(), SRC_DEF_ID, "src", JSONB.valueOf("{}"), ActorType.source) - .values(dstId, UUID.randomUUID(), DST_DEF_ID, "dst", JSONB.valueOf("{}"), ActorType.destination) + .values(srcId, UUID.randomUUID(), SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source) + .values(dstId, UUID.randomUUID(), DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination) .execute()); final var connId = UUID.randomUUID(); // create connection @@ -146,7 +150,7 @@ void shouldReturnReleaseStages() throws SQLException { ctx -> ctx .insertInto(CONNECTION, CONNECTION.ID, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, CONNECTION.NAME, CONNECTION.CATALOG, CONNECTION.MANUAL) - .values(connId, NamespaceDefinitionType.source, srcId, dstId, "conn", JSONB.valueOf("{}"), true) + .values(connId, NamespaceDefinitionType.source, srcId, dstId, CONN, JSONB.valueOf("{}"), true) .execute()); // create job final var jobId = 1L; @@ -158,7 +162,7 @@ void shouldReturnReleaseStages() throws SQLException { } @Test - @DisplayName("should not error out or return any result if not applicable") + @DisplayName(DISPLAY_NAME) void shouldReturnNothingIfNotApplicable() throws SQLException { final var missingJobId = 100000L; final var res = configDb.query(ctx -> MetricQueries.jobIdToReleaseStages(ctx, missingJobId)); @@ -181,8 +185,8 @@ void runningJobsShouldReturnCorrectCount() throws SQLException { final var dstId = UUID.randomUUID(); configDb.transaction( ctx -> ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE) - .values(srcId, UUID.randomUUID(), SRC_DEF_ID, "src", JSONB.valueOf("{}"), ActorType.source) - .values(dstId, UUID.randomUUID(), DST_DEF_ID, "dst", JSONB.valueOf("{}"), ActorType.destination) + .values(srcId, UUID.randomUUID(), SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source) + .values(dstId, UUID.randomUUID(), DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination) .execute()); final UUID activeConnectionId = UUID.randomUUID(); final UUID inactiveConnectionId = UUID.randomUUID(); @@ -190,8 +194,8 @@ void runningJobsShouldReturnCorrectCount() throws SQLException { ctx -> ctx .insertInto(CONNECTION, CONNECTION.ID, CONNECTION.STATUS, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, CONNECTION.NAME, CONNECTION.CATALOG, CONNECTION.MANUAL) - .values(activeConnectionId, StatusType.active, NamespaceDefinitionType.source, srcId, dstId, "conn", JSONB.valueOf("{}"), true) - .values(inactiveConnectionId, StatusType.inactive, NamespaceDefinitionType.source, srcId, dstId, "conn", JSONB.valueOf("{}"), true) + .values(activeConnectionId, StatusType.active, NamespaceDefinitionType.source, srcId, dstId, CONN, JSONB.valueOf("{}"), true) + .values(inactiveConnectionId, StatusType.inactive, NamespaceDefinitionType.source, srcId, dstId, CONN, JSONB.valueOf("{}"), true) .execute()); // non-pending jobs @@ -284,7 +288,7 @@ void shouldReturnOnlyPendingSeconds() throws SQLException { } @Test - @DisplayName("should not error out or return any result if not applicable") + @DisplayName(DISPLAY_NAME) void shouldReturnNothingIfNotApplicable() throws SQLException { configDb.transaction( ctx -> ctx.insertInto(JOBS, JOBS.ID, JOBS.SCOPE, JOBS.STATUS).values(1L, "", JobStatus.succeeded).execute()); @@ -331,7 +335,7 @@ void shouldReturnOnlyRunningSeconds() throws SQLException { } @Test - @DisplayName("should not error out or return any result if not applicable") + @DisplayName(DISPLAY_NAME) void shouldReturnNothingIfNotApplicable() throws SQLException { configDb.transaction( ctx -> ctx.insertInto(JOBS, JOBS.ID, JOBS.SCOPE, JOBS.STATUS).values(1L, "", JobStatus.succeeded).execute()); @@ -370,16 +374,16 @@ void shouldReturnNumConnectionsBasic() throws SQLException { ctx -> ctx .insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE, ACTOR.TOMBSTONE) - .values(srcId, workspaceId, SRC_DEF_ID, "src", JSONB.valueOf("{}"), ActorType.source, false) - .values(dstId, workspaceId, DST_DEF_ID, "dst", JSONB.valueOf("{}"), ActorType.destination, false) + .values(srcId, workspaceId, SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) + .values(dstId, workspaceId, DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) .execute()); configDb.transaction( ctx -> ctx .insertInto(CONNECTION, CONNECTION.ID, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, CONNECTION.NAME, CONNECTION.CATALOG, CONNECTION.MANUAL, CONNECTION.STATUS) - .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, "conn", JSONB.valueOf("{}"), true, StatusType.active) - .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, "conn", JSONB.valueOf("{}"), true, StatusType.active) + .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, CONN, JSONB.valueOf("{}"), true, StatusType.active) + .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, CONN, JSONB.valueOf("{}"), true, StatusType.active) .execute()); final var res = configDb.query(MetricQueries::numberOfActiveConnPerWorkspace); @@ -401,18 +405,18 @@ void shouldIgnoreNonRunningConnections() throws SQLException { ctx -> ctx .insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE, ACTOR.TOMBSTONE) - .values(srcId, workspaceId, SRC_DEF_ID, "src", JSONB.valueOf("{}"), ActorType.source, false) - .values(dstId, workspaceId, DST_DEF_ID, "dst", JSONB.valueOf("{}"), ActorType.destination, false) + .values(srcId, workspaceId, SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) + .values(dstId, workspaceId, DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) .execute()); configDb.transaction( ctx -> ctx .insertInto(CONNECTION, CONNECTION.ID, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, CONNECTION.NAME, CONNECTION.CATALOG, CONNECTION.MANUAL, CONNECTION.STATUS) - .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, "conn", JSONB.valueOf("{}"), true, StatusType.active) - .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, "conn", JSONB.valueOf("{}"), true, StatusType.active) - .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, "conn", JSONB.valueOf("{}"), true, StatusType.deprecated) - .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, "conn", JSONB.valueOf("{}"), true, StatusType.inactive) + .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, CONN, JSONB.valueOf("{}"), true, StatusType.active) + .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, CONN, JSONB.valueOf("{}"), true, StatusType.active) + .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, CONN, JSONB.valueOf("{}"), true, StatusType.deprecated) + .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, CONN, JSONB.valueOf("{}"), true, StatusType.inactive) .execute()); final var res = configDb.query(MetricQueries::numberOfActiveConnPerWorkspace); @@ -434,15 +438,15 @@ void shouldIgnoreDeletedWorkspaces() throws SQLException { ctx -> ctx .insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE, ACTOR.TOMBSTONE) - .values(srcId, workspaceId, SRC_DEF_ID, "src", JSONB.valueOf("{}"), ActorType.source, false) - .values(dstId, workspaceId, DST_DEF_ID, "dst", JSONB.valueOf("{}"), ActorType.destination, false) + .values(srcId, workspaceId, SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) + .values(dstId, workspaceId, DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) .execute()); configDb.transaction( ctx -> ctx .insertInto(CONNECTION, CONNECTION.ID, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, CONNECTION.NAME, CONNECTION.CATALOG, CONNECTION.MANUAL, CONNECTION.STATUS) - .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, "conn", JSONB.valueOf("{}"), true, StatusType.active) + .values(UUID.randomUUID(), NamespaceDefinitionType.source, srcId, dstId, CONN, JSONB.valueOf("{}"), true, StatusType.active) .execute()); final var res = configDb.query(MetricQueries::numberOfActiveConnPerWorkspace); @@ -450,7 +454,7 @@ void shouldIgnoreDeletedWorkspaces() throws SQLException { } @Test - @DisplayName("should not error out or return any result if not applicable") + @DisplayName(DISPLAY_NAME) void shouldReturnNothingIfNotApplicable() throws SQLException { final var res = configDb.query(MetricQueries::numberOfActiveConnPerWorkspace); assertEquals(0, res.size()); @@ -554,7 +558,7 @@ void shouldReturnTerminalJobsComplex() throws SQLException { } @Test - @DisplayName("should not error out or return any result if not applicable") + @DisplayName(DISPLAY_NAME) void shouldReturnNothingIfNotApplicable() throws SQLException { final var res = configDb.query(MetricQueries::overallJobRuntimeForTerminalJobsInLastHour); assertEquals(0, res.size()); diff --git a/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java b/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java index 6795cfd8871d..aecb3c757321 100644 --- a/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java +++ b/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java @@ -35,6 +35,11 @@ class SlackNotificationClientTest { private static final Logger LOGGER = LoggerFactory.getLogger(SlackNotificationClientTest.class); private static final UUID WORKSPACE_ID = UUID.randomUUID(); private static final UUID CONNECTION_ID = UUID.randomUUID(); + private static final String TEST_PATH = "/test"; + private static final String DESTINATION_TEST = "destination-test"; + private static final String JOB_DESCRIPTION = "job description"; + private static final String LOG_URL = "logUrl"; + private static final String SOURCE_TEST = "source-test"; public static final String WEBHOOK_URL = "http://localhost:"; private static final String EXPECTED_FAIL_MESSAGE = "Your connection from source-test to destination-test just failed...\n" @@ -62,11 +67,11 @@ void tearDown() { @Test void testBadResponseWrongNotificationMessage() throws IOException, InterruptedException { final String message = UUID.randomUUID().toString(); - server.createContext("/test", new ServerHandler("Message mismatched")); + server.createContext(TEST_PATH, new ServerHandler("Message mismatched")); final SlackNotificationClient client = new SlackNotificationClient(new Notification() .withNotificationType(NotificationType.SLACK) - .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + "/test"))); + .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + TEST_PATH))); assertThrows(IOException.class, () -> client.notifyFailure(message)); } @@ -76,7 +81,7 @@ void testBadWebhookUrl() { new SlackNotificationClient(new Notification() .withNotificationType(NotificationType.SLACK) .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + "/bad"))); - assertThrows(IOException.class, () -> client.notifyJobFailure("source-test", "destination-test", "job description", "logUrl")); + assertThrows(IOException.class, () -> client.notifyJobFailure(SOURCE_TEST, DESTINATION_TEST, JOB_DESCRIPTION, LOG_URL)); } @Test @@ -84,40 +89,40 @@ void testEmptyWebhookUrl() throws IOException, InterruptedException { final SlackNotificationClient client = new SlackNotificationClient( new Notification().withNotificationType(NotificationType.SLACK).withSlackConfiguration(new SlackNotificationConfiguration())); - assertFalse(client.notifyJobFailure("source-test", "destination-test", "job description", "logUrl")); + assertFalse(client.notifyJobFailure(SOURCE_TEST, DESTINATION_TEST, JOB_DESCRIPTION, LOG_URL)); } @Test void testNotify() throws IOException, InterruptedException { final String message = UUID.randomUUID().toString(); - server.createContext("/test", new ServerHandler(message)); + server.createContext(TEST_PATH, new ServerHandler(message)); final SlackNotificationClient client = new SlackNotificationClient(new Notification() .withNotificationType(NotificationType.SLACK) - .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + "/test"))); + .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + TEST_PATH))); assertTrue(client.notifyFailure(message)); assertFalse(client.notifySuccess(message)); } @Test void testNotifyJobFailure() throws IOException, InterruptedException { - server.createContext("/test", new ServerHandler(EXPECTED_FAIL_MESSAGE)); + server.createContext(TEST_PATH, new ServerHandler(EXPECTED_FAIL_MESSAGE)); final SlackNotificationClient client = new SlackNotificationClient(new Notification() .withNotificationType(NotificationType.SLACK) - .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + "/test"))); - assertTrue(client.notifyJobFailure("source-test", "destination-test", "job description", "logUrl")); + .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + TEST_PATH))); + assertTrue(client.notifyJobFailure(SOURCE_TEST, DESTINATION_TEST, JOB_DESCRIPTION, LOG_URL)); } @Test void testNotifyJobSuccess() throws IOException, InterruptedException { - server.createContext("/test", new ServerHandler(EXPECTED_SUCCESS_MESSAGE)); + server.createContext(TEST_PATH, new ServerHandler(EXPECTED_SUCCESS_MESSAGE)); final SlackNotificationClient client = new SlackNotificationClient(new Notification() .withNotificationType(NotificationType.SLACK) .withSendOnSuccess(true) - .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + "/test"))); - assertTrue(client.notifyJobSuccess("source-test", "destination-test", "job description", "logUrl")); + .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + TEST_PATH))); + assertTrue(client.notifyJobSuccess(SOURCE_TEST, DESTINATION_TEST, JOB_DESCRIPTION, LOG_URL)); } @Test @@ -133,13 +138,13 @@ void testNotifyConnectionDisabled() throws IOException, InterruptedException { """, WORKSPACE_ID, CONNECTION_ID); - server.createContext("/test", new ServerHandler(expectedNotificationMessage)); + server.createContext(TEST_PATH, new ServerHandler(expectedNotificationMessage)); final SlackNotificationClient client = new SlackNotificationClient(new Notification() .withNotificationType(NotificationType.SLACK) .withSendOnSuccess(true) - .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + "/test"))); - assertTrue(client.notifyConnectionDisabled("", "source-test", "destination-test", "job description.", WORKSPACE_ID, CONNECTION_ID)); + .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + TEST_PATH))); + assertTrue(client.notifyConnectionDisabled("", SOURCE_TEST, DESTINATION_TEST, "job description.", WORKSPACE_ID, CONNECTION_ID)); } @Test @@ -155,13 +160,13 @@ void testNotifyConnectionDisabledWarning() throws IOException, InterruptedExcept """, WORKSPACE_ID, CONNECTION_ID); - server.createContext("/test", new ServerHandler(expectedNotificationWarningMessage)); + server.createContext(TEST_PATH, new ServerHandler(expectedNotificationWarningMessage)); final SlackNotificationClient client = new SlackNotificationClient(new Notification() .withNotificationType(NotificationType.SLACK) .withSendOnSuccess(true) - .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + "/test"))); - assertTrue(client.notifyConnectionDisableWarning("", "source-test", "destination-test", "job description.", WORKSPACE_ID, CONNECTION_ID)); + .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + TEST_PATH))); + assertTrue(client.notifyConnectionDisableWarning("", SOURCE_TEST, DESTINATION_TEST, "job description.", WORKSPACE_ID, CONNECTION_ID)); } static class ServerHandler implements HttpHandler { diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/DriftOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/DriftOAuthFlow.java index b971e44a58ea..aec985fb83b7 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/DriftOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/DriftOAuthFlow.java @@ -25,6 +25,7 @@ public class DriftOAuthFlow extends BaseOAuth2Flow { private static final String ACCESS_TOKEN_URL = "https://driftapi.com/oauth2/token"; + private static final String CODE = "code"; public DriftOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { super(configRepository, httpClient); @@ -42,7 +43,7 @@ protected String formatConsentUrl(final UUID definitionId, final String clientId .setScheme("https") .setHost("dev.drift.com") .setPath("authorize") - .addParameter("response_type", "code") + .addParameter("response_type", CODE) .addParameter("client_id", clientId) .addParameter("redirect_uri", redirectUrl) .addParameter("state", getState()); @@ -55,8 +56,8 @@ protected String formatConsentUrl(final UUID definitionId, final String clientId @Override protected String extractCodeParameter(final Map queryParams) throws IOException { - if (queryParams.containsKey("code")) { - return (String) queryParams.get("code"); + if (queryParams.containsKey(CODE)) { + return (String) queryParams.get(CODE); } else { throw new IOException("Undefined 'code' from consent redirected url."); } @@ -75,7 +76,7 @@ protected Map getAccessTokenQueryParameters(final String clientI return ImmutableMap.builder() .put("client_id", clientId) .put("client_secret", clientSecret) - .put("code", authCode) + .put(CODE, authCode) .put("grant_type", "authorization_code") .build(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/TrelloOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/TrelloOAuthFlow.java index 0f607e9b023b..57331ffb7232 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/TrelloOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/TrelloOAuthFlow.java @@ -32,6 +32,7 @@ public class TrelloOAuthFlow extends BaseOAuthFlow { private static final String REQUEST_TOKEN_URL = "https://trello.com/1/OAuthGetRequestToken"; private static final String AUTHENTICATE_URL = "https://trello.com/1/OAuthAuthorizeToken"; private static final String ACCESS_TOKEN_URL = "https://trello.com/1/OAuthGetAccessToken"; + private static final String OAUTH_VERIFIER = "oauth_verifier"; // Airbyte webserver creates new TrelloOAuthFlow class instance for every API // call. Since oAuth 1.0 workflow requires data from previous step to build @@ -141,12 +142,12 @@ public Map completeDestinationOAuth(final UUID workspaceId, private Map internalCompleteOAuth(final JsonNode oAuthParamConfig, final Map queryParams) throws IOException { final String clientKey = getClientIdUnsafe(oAuthParamConfig); - if (!queryParams.containsKey("oauth_verifier") || !queryParams.containsKey("oauth_token")) { + if (!queryParams.containsKey(OAUTH_VERIFIER) || !queryParams.containsKey("oauth_token")) { throw new IOException( - "Undefined " + (!queryParams.containsKey("oauth_verifier") ? "oauth_verifier" : "oauth_token") + " from consent redirected url."); + "Undefined " + (!queryParams.containsKey(OAUTH_VERIFIER) ? OAUTH_VERIFIER : "oauth_token") + " from consent redirected url."); } final String temporaryToken = (String) queryParams.get("oauth_token"); - final String verificationCode = (String) queryParams.get("oauth_verifier"); + final String verificationCode = (String) queryParams.get(OAUTH_VERIFIER); final OAuthGetAccessToken oAuthGetAccessToken = new OAuthGetAccessToken(ACCESS_TOKEN_URL); oAuthGetAccessToken.signer = signer; oAuthGetAccessToken.transport = transport; diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlow.java index 97378ebed5c8..a6f300fbc8e2 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/facebook/FacebookOAuthFlow.java @@ -29,6 +29,7 @@ public abstract class FacebookOAuthFlow extends BaseOAuth2Flow { private static final String ACCESS_TOKEN_URL = "https://graph.facebook.com/v12.0/oauth/access_token"; private static final String AUTH_CODE_TOKEN_URL = "https://www.facebook.com/v12.0/dialog/oauth"; + private static final String ACCESS_TOKEN = "access_token"; public FacebookOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { super(configRepository, httpClient); @@ -67,8 +68,8 @@ protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) { protected Map extractOAuthOutput(final JsonNode data, final String accessTokenUrl) { // Facebook does not have refresh token but calls it "long lived access token" instead: // see https://developers.facebook.com/docs/facebook-login/access-tokens/refreshing - Preconditions.checkArgument(data.has("access_token"), "Missing 'access_token' in query params from %s", ACCESS_TOKEN_URL); - return Map.of("access_token", data.get("access_token").asText()); + Preconditions.checkArgument(data.has(ACCESS_TOKEN), "Missing 'access_token' in query params from %s", ACCESS_TOKEN_URL); + return Map.of(ACCESS_TOKEN, data.get(ACCESS_TOKEN).asText()); } @Override @@ -87,10 +88,10 @@ protected Map completeOAuthFlow(final String clientId, final Map data = super.completeOAuthFlow(clientId, clientSecret, authCode, redirectUrl, inputOAuthConfiguration, oAuthParamConfig); - Preconditions.checkArgument(data.containsKey("access_token")); - final String shortLivedAccessToken = (String) data.get("access_token"); + Preconditions.checkArgument(data.containsKey(ACCESS_TOKEN)); + final String shortLivedAccessToken = (String) data.get(ACCESS_TOKEN); final String longLivedAccessToken = getLongLivedAccessToken(clientId, clientSecret, shortLivedAccessToken); - return Map.of("access_token", longLivedAccessToken); + return Map.of(ACCESS_TOKEN, longLivedAccessToken); } protected URI createLongLivedTokenURI(final String clientId, final String clientSecret, final String shortLivedAccessToken) @@ -118,8 +119,8 @@ protected String getLongLivedAccessToken(final String clientId, final String cli .build(); final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); final JsonNode responseJson = Jsons.deserialize(response.body()); - Preconditions.checkArgument(responseJson.hasNonNull("access_token"), "%s response should have access_token", responseJson); - return responseJson.get("access_token").asText(); + Preconditions.checkArgument(responseJson.hasNonNull(ACCESS_TOKEN), "%s response should have access_token", responseJson); + return responseJson.get(ACCESS_TOKEN).asText(); } catch (final InterruptedException | URISyntaxException e) { throw new IOException("Failed to complete OAuth flow", e); } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/MoreOAuthParametersTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/MoreOAuthParametersTest.java index 4d442d58c912..b8a60daf403a 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/MoreOAuthParametersTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/MoreOAuthParametersTest.java @@ -18,14 +18,17 @@ class MoreOAuthParametersTest { + private static final String FIELD = "field"; + private static final String OAUTH_CREDS = "oauth_credentials"; + @Test void testFlattenConfig() { final JsonNode nestedConfig = Jsons.jsonNode(Map.of( - "field", "value1", + FIELD, "value1", "top-level", Map.of( "nested_field", "value2"))); final JsonNode expectedConfig = Jsons.jsonNode(Map.of( - "field", "value1", + FIELD, "value1", "nested_field", "value2")); final JsonNode actualConfig = MoreOAuthParameters.flattenOAuthConfig(nestedConfig); assertEquals(expectedConfig, actualConfig); @@ -34,10 +37,10 @@ void testFlattenConfig() { @Test void testFailureFlattenConfig() { final JsonNode nestedConfig = Jsons.jsonNode(Map.of( - "field", "value1", + FIELD, "value1", "top-level", Map.of( "nested_field", "value2", - "field", "value3"))); + FIELD, "value3"))); assertThrows(IllegalStateException.class, () -> MoreOAuthParameters.flattenOAuthConfig(nestedConfig)); } @@ -59,13 +62,13 @@ void testInjectUnnestedNode() { void testInjectNewNestedNode() { final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); final ObjectNode nestedConfig = (ObjectNode) Jsons.jsonNode(ImmutableMap.builder() - .put("oauth_credentials", oauthParams) + .put(OAUTH_CREDS, oauthParams) .build()); // nested node does not exist in actual object final ObjectNode actual = generateJsonConfig(); final ObjectNode expected = Jsons.clone(actual); - expected.putObject("oauth_credentials").setAll(oauthParams); + expected.putObject(OAUTH_CREDS).setAll(oauthParams); MoreOAuthParameters.mergeJsons(actual, nestedConfig); @@ -77,14 +80,14 @@ void testInjectNewNestedNode() { void testInjectedPartiallyExistingNestedNode() { final ObjectNode oauthParams = (ObjectNode) Jsons.jsonNode(generateOAuthParameters()); final ObjectNode nestedConfig = (ObjectNode) Jsons.jsonNode(ImmutableMap.builder() - .put("oauth_credentials", oauthParams) + .put(OAUTH_CREDS, oauthParams) .build()); // nested node partially exists in actual object final ObjectNode actual = generateJsonConfig(); - actual.putObject("oauth_credentials").put("irrelevant_field", "_"); + actual.putObject(OAUTH_CREDS).put("irrelevant_field", "_"); final ObjectNode expected = Jsons.clone(actual); - ((ObjectNode) expected.get("oauth_credentials")).setAll(oauthParams); + ((ObjectNode) expected.get(OAUTH_CREDS)).setAll(oauthParams); MoreOAuthParameters.mergeJsons(actual, nestedConfig); diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/BaseOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/BaseOAuthFlowTest.java index 9400588a9731..fd146be96c9a 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/BaseOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/BaseOAuthFlowTest.java @@ -34,6 +34,12 @@ public abstract class BaseOAuthFlowTest { private static final String REDIRECT_URL = "https://airbyte.io"; + private static final String REFRESH_TOKEN = "refresh_token"; + private static final String CLIENT_ID = "client_id"; + private static final String TYPE = "type"; + private static final String CODE = "code"; + private static final String TEST_CODE = "test_code"; + private static final String EXPECTED_BUT_GOT = "Expected %s values but got\n\t%s\ninstead of\n\t%s"; private HttpClient httpClient; private ConfigRepository configRepository; @@ -93,8 +99,8 @@ void setup() throws JsonValidationException, IOException { */ protected Map getExpectedOutput() { return Map.of( - "refresh_token", "refresh_token_response", - "client_id", MoreOAuthParameters.SECRET_MASK, + REFRESH_TOKEN, "refresh_token_response", + CLIENT_ID, MoreOAuthParameters.SECRET_MASK, "client_secret", MoreOAuthParameters.SECRET_MASK); } @@ -105,7 +111,7 @@ protected Map getExpectedOutput() { * @return the output specification used to identify what the oauth flow should be returning */ protected JsonNode getCompleteOAuthOutputSpecification() { - return getJsonSchema(Map.of("refresh_token", Map.of("type", "string"))); + return getJsonSchema(Map.of(REFRESH_TOKEN, Map.of(TYPE, "string"))); } /** @@ -116,15 +122,15 @@ protected JsonNode getCompleteOAuthOutputSpecification() { */ protected Map getExpectedFilteredOutput() { return Map.of( - "refresh_token", "refresh_token_response", - "client_id", MoreOAuthParameters.SECRET_MASK); + REFRESH_TOKEN, "refresh_token_response", + CLIENT_ID, MoreOAuthParameters.SECRET_MASK); } /** * @return the output specification used to filter what the oauth flow should be returning */ protected JsonNode getCompleteOAuthServerOutputSpecification() { - return getJsonSchema(Map.of("client_id", Map.of("type", "string"))); + return getJsonSchema(Map.of(CLIENT_ID, Map.of(TYPE, "string"))); } /** @@ -168,14 +174,14 @@ protected JsonNode getUserInputFromConnectorConfigSpecification() { */ protected JsonNode getOAuthParamConfig() { return Jsons.jsonNode(ImmutableMap.builder() - .put("client_id", "test_client_id") + .put(CLIENT_ID, "test_client_id") .put("client_secret", "test_client_secret") .build()); } protected static JsonNode getJsonSchema(final Map properties) { return Jsons.jsonNode(Map.of( - "type", "object", + TYPE, "object", "additionalProperties", "false", "properties", properties)); } @@ -205,8 +211,8 @@ protected String getMockedResponse() { protected OAuthConfigSpecification getOAuthConfigSpecification() { return getoAuthConfigSpecification() // change property types to induce json validation errors. - .withCompleteOauthServerOutputSpecification(getJsonSchema(Map.of("client_id", Map.of("type", "integer")))) - .withCompleteOauthOutputSpecification(getJsonSchema(Map.of("refresh_token", Map.of("type", "integer")))); + .withCompleteOauthServerOutputSpecification(getJsonSchema(Map.of(CLIENT_ID, Map.of(TYPE, "integer")))) + .withCompleteOauthOutputSpecification(getJsonSchema(Map.of(REFRESH_TOKEN, Map.of(TYPE, "integer")))); } @Test @@ -305,7 +311,7 @@ void testDeprecatedCompleteSourceOAuth() throws IOException, InterruptedExceptio final HttpResponse response = mock(HttpResponse.class); when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); + final Map queryParams = Map.of(CODE, TEST_CODE); if (hasDependencyOnConnectorConfigValues()) { assertThrows(IOException.class, () -> oauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL), @@ -319,7 +325,7 @@ void testDeprecatedCompleteSourceOAuth() throws IOException, InterruptedExceptio final Map expectedOutput = returnedCredentials; final Map actualQueryParams = actualRawQueryParams; assertEquals(expectedOutput.size(), actualQueryParams.size(), - String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + String.format(EXPECTED_BUT_GOT, expectedOutput.size(), actualQueryParams, expectedOutput)); expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); } } @@ -330,7 +336,7 @@ void testDeprecatedCompleteDestinationOAuth() throws IOException, ConfigNotFound final HttpResponse response = mock(HttpResponse.class); when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); + final Map queryParams = Map.of(CODE, TEST_CODE); if (hasDependencyOnConnectorConfigValues()) { assertThrows(IOException.class, () -> oauthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL), @@ -344,7 +350,7 @@ void testDeprecatedCompleteDestinationOAuth() throws IOException, ConfigNotFound final Map expectedOutput = returnedCredentials; final Map actualQueryParams = actualRawQueryParams; assertEquals(expectedOutput.size(), actualQueryParams.size(), - String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + String.format(EXPECTED_BUT_GOT, expectedOutput.size(), actualQueryParams, expectedOutput)); expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); } } @@ -354,7 +360,7 @@ void testEmptyOutputCompleteSourceOAuth() throws IOException, InterruptedExcepti final HttpResponse response = mock(HttpResponse.class); when(response.body()).thenReturn(getMockedResponse()); when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); + final Map queryParams = Map.of(CODE, TEST_CODE); final Map actualQueryParams = oauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, getInputOAuthConfiguration(), getEmptyOAuthConfigSpecification()); assertEquals(0, actualQueryParams.size(), @@ -366,7 +372,7 @@ void testEmptyOutputCompleteDestinationOAuth() throws IOException, InterruptedEx final HttpResponse response = mock(HttpResponse.class); when(response.body()).thenReturn(getMockedResponse()); when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); + final Map queryParams = Map.of(CODE, TEST_CODE); final Map actualQueryParams = oauthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, getInputOAuthConfiguration(), getEmptyOAuthConfigSpecification()); assertEquals(0, actualQueryParams.size(), @@ -378,12 +384,12 @@ void testEmptyInputCompleteSourceOAuth() throws IOException, InterruptedExceptio final HttpResponse response = mock(HttpResponse.class); when(response.body()).thenReturn(getMockedResponse()); when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); + final Map queryParams = Map.of(CODE, TEST_CODE); final Map actualQueryParams = oauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, Jsons.emptyObject(), getoAuthConfigSpecification()); final Map expectedOutput = getExpectedFilteredOutput(); assertEquals(expectedOutput.size(), actualQueryParams.size(), - String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + String.format(EXPECTED_BUT_GOT, expectedOutput.size(), actualQueryParams, expectedOutput)); expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); } @@ -392,12 +398,12 @@ void testEmptyInputCompleteDestinationOAuth() throws IOException, InterruptedExc final HttpResponse response = mock(HttpResponse.class); when(response.body()).thenReturn(getMockedResponse()); when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); + final Map queryParams = Map.of(CODE, TEST_CODE); final Map actualQueryParams = oauthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, Jsons.emptyObject(), getoAuthConfigSpecification()); final Map expectedOutput = getExpectedFilteredOutput(); assertEquals(expectedOutput.size(), actualQueryParams.size(), - String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + String.format(EXPECTED_BUT_GOT, expectedOutput.size(), actualQueryParams, expectedOutput)); expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); } @@ -406,12 +412,12 @@ void testCompleteSourceOAuth() throws IOException, InterruptedException, ConfigN final HttpResponse response = mock(HttpResponse.class); when(response.body()).thenReturn(getMockedResponse()); when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); + final Map queryParams = Map.of(CODE, TEST_CODE); final Map actualQueryParams = oauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, getInputOAuthConfiguration(), getoAuthConfigSpecification()); final Map expectedOutput = getExpectedFilteredOutput(); assertEquals(expectedOutput.size(), actualQueryParams.size(), - String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + String.format(EXPECTED_BUT_GOT, expectedOutput.size(), actualQueryParams, expectedOutput)); expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); } @@ -420,12 +426,12 @@ void testCompleteDestinationOAuth() throws IOException, InterruptedException, Co final HttpResponse response = mock(HttpResponse.class); when(response.body()).thenReturn(getMockedResponse()); when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); + final Map queryParams = Map.of(CODE, TEST_CODE); final Map actualQueryParams = oauthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, getInputOAuthConfiguration(), getoAuthConfigSpecification()); final Map expectedOutput = getExpectedFilteredOutput(); assertEquals(expectedOutput.size(), actualQueryParams.size(), - String.format("Expected %s values but got\n\t%s\ninstead of\n\t%s", expectedOutput.size(), actualQueryParams, expectedOutput)); + String.format(EXPECTED_BUT_GOT, expectedOutput.size(), actualQueryParams, expectedOutput)); expectedOutput.forEach((key, value) -> assertEquals(value, actualQueryParams.get(key))); } @@ -434,7 +440,7 @@ void testValidateOAuthOutputFailure() throws IOException, InterruptedException, final HttpResponse response = mock(HttpResponse.class); when(response.body()).thenReturn(getMockedResponse()); when(httpClient.send(any(), any())).thenReturn(response); - final Map queryParams = Map.of("code", "test_code"); + final Map queryParams = Map.of(CODE, TEST_CODE); final OAuthConfigSpecification oAuthConfigSpecification = getOAuthConfigSpecification(); assertThrows(JsonValidationException.class, () -> oauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL, getInputOAuthConfiguration(), oAuthConfigSpecification)); diff --git a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index 092e36845876..a7777fa1ffe5 100644 --- a/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -26,6 +26,10 @@ class CatalogHelpersTest { // handy for debugging test only. private static final Comparator STREAM_TRANSFORM_COMPARATOR = Comparator.comparing(StreamTransform::getTransformType); + private static final String CAD = "CAD"; + private static final String ITEMS = "items"; + private static final String SOME_ARRAY = "someArray"; + private static final String PROPERTIES = "properties"; @Test void testFieldToJsonSchema() { @@ -82,7 +86,7 @@ void testGetFieldNames() throws IOException { final JsonNode node = Jsons.deserialize(MoreResources.readResource("valid_schema.json")); final Set actualFieldNames = CatalogHelpers.getAllFieldNames(node); final List expectedFieldNames = - List.of("CAD", "DKK", "HKD", "HUF", "ISK", "PHP", "date", "nestedkey", "somekey", "something", "something2", "æ–‡", "someArray", "items", + List.of(CAD, "DKK", "HKD", "HUF", "ISK", "PHP", "date", "nestedkey", "somekey", "something", "something2", "æ–‡", SOME_ARRAY, ITEMS, "oldName"); // sort so that the diff is easier to read. @@ -105,22 +109,22 @@ void testGetCatalogDiff() throws IOException { StreamTransform.createAddStreamTransform(new StreamDescriptor().withName("sales")), StreamTransform.createRemoveStreamTransform(new StreamDescriptor().withName("accounts")), StreamTransform.createUpdateStreamTransform(new StreamDescriptor().withName("users"), new UpdateStreamTransform(Set.of( - FieldTransform.createAddFieldTransform(List.of("COD"), schema2.get("properties").get("COD")), - FieldTransform.createRemoveFieldTransform(List.of("something2"), schema1.get("properties").get("something2")), - FieldTransform.createRemoveFieldTransform(List.of("HKD"), schema1.get("properties").get("HKD")), - FieldTransform.createUpdateFieldTransform(List.of("CAD"), new UpdateFieldSchemaTransform( - schema1.get("properties").get("CAD"), - schema2.get("properties").get("CAD"))), - FieldTransform.createUpdateFieldTransform(List.of("someArray"), new UpdateFieldSchemaTransform( - schema1.get("properties").get("someArray"), - schema2.get("properties").get("someArray"))), - FieldTransform.createUpdateFieldTransform(List.of("someArray", "items"), new UpdateFieldSchemaTransform( - schema1.get("properties").get("someArray").get("items"), - schema2.get("properties").get("someArray").get("items"))), - FieldTransform.createRemoveFieldTransform(List.of("someArray", "items", "oldName"), - schema1.get("properties").get("someArray").get("items").get("properties").get("oldName")), - FieldTransform.createAddFieldTransform(List.of("someArray", "items", "newName"), - schema2.get("properties").get("someArray").get("items").get("properties").get("newName")))))) + FieldTransform.createAddFieldTransform(List.of("COD"), schema2.get(PROPERTIES).get("COD")), + FieldTransform.createRemoveFieldTransform(List.of("something2"), schema1.get(PROPERTIES).get("something2")), + FieldTransform.createRemoveFieldTransform(List.of("HKD"), schema1.get(PROPERTIES).get("HKD")), + FieldTransform.createUpdateFieldTransform(List.of(CAD), new UpdateFieldSchemaTransform( + schema1.get(PROPERTIES).get(CAD), + schema2.get(PROPERTIES).get(CAD))), + FieldTransform.createUpdateFieldTransform(List.of(SOME_ARRAY), new UpdateFieldSchemaTransform( + schema1.get(PROPERTIES).get(SOME_ARRAY), + schema2.get(PROPERTIES).get(SOME_ARRAY))), + FieldTransform.createUpdateFieldTransform(List.of(SOME_ARRAY, ITEMS), new UpdateFieldSchemaTransform( + schema1.get(PROPERTIES).get(SOME_ARRAY).get(ITEMS), + schema2.get(PROPERTIES).get(SOME_ARRAY).get(ITEMS))), + FieldTransform.createRemoveFieldTransform(List.of(SOME_ARRAY, ITEMS, "oldName"), + schema1.get(PROPERTIES).get(SOME_ARRAY).get(ITEMS).get(PROPERTIES).get("oldName")), + FieldTransform.createAddFieldTransform(List.of(SOME_ARRAY, ITEMS, "newName"), + schema2.get(PROPERTIES).get(SOME_ARRAY).get(ITEMS).get(PROPERTIES).get("newName")))))) .sorted(STREAM_TRANSFORM_COMPARATOR) .toList(); assertEquals(expectedDiff, actualDiff.stream().sorted(STREAM_TRANSFORM_COMPARATOR).toList()); diff --git a/airbyte-queue/src/test/java/io/airbyte/queue/OnDiskQueueTest.java b/airbyte-queue/src/test/java/io/airbyte/queue/OnDiskQueueTest.java index ad8317afffaf..be9d1e5c7798 100644 --- a/airbyte-queue/src/test/java/io/airbyte/queue/OnDiskQueueTest.java +++ b/airbyte-queue/src/test/java/io/airbyte/queue/OnDiskQueueTest.java @@ -23,6 +23,7 @@ class OnDiskQueueTest { private static final Path TEST_ROOT = Path.of("/tmp/airbyte_tests"); + private static final String HELLO = "hello"; private CloseableQueue queue; private Path queueRoot; @@ -39,24 +40,24 @@ void teardown() throws Exception { @Test void testPoll() { - queue.offer(toBytes("hello")); - assertEquals("hello", new String(Objects.requireNonNull(queue.poll()), Charsets.UTF_8)); + queue.offer(toBytes(HELLO)); + assertEquals(HELLO, new String(Objects.requireNonNull(queue.poll()), Charsets.UTF_8)); } @Test void testPeek() { - queue.offer(toBytes("hello")); - assertEquals("hello", new String(Objects.requireNonNull(queue.peek()), Charsets.UTF_8)); - assertEquals("hello", new String(Objects.requireNonNull(queue.peek()), Charsets.UTF_8)); - assertEquals("hello", new String(Objects.requireNonNull(queue.poll()), Charsets.UTF_8)); + queue.offer(toBytes(HELLO)); + assertEquals(HELLO, new String(Objects.requireNonNull(queue.peek()), Charsets.UTF_8)); + assertEquals(HELLO, new String(Objects.requireNonNull(queue.peek()), Charsets.UTF_8)); + assertEquals(HELLO, new String(Objects.requireNonNull(queue.poll()), Charsets.UTF_8)); } @Test void testSize() { assertEquals(0, queue.size()); - queue.offer(toBytes("hello")); + queue.offer(toBytes(HELLO)); assertEquals(1, queue.size()); - queue.offer(toBytes("hello")); + queue.offer(toBytes(HELLO)); assertEquals(2, queue.size()); } @@ -64,7 +65,7 @@ void testSize() { void testClosed() throws Exception { queue.close(); assertDoesNotThrow(() -> queue.close()); - assertThrows(IllegalStateException.class, () -> queue.offer(toBytes("hello"))); + assertThrows(IllegalStateException.class, () -> queue.offer(toBytes(HELLO))); assertThrows(IllegalStateException.class, () -> queue.poll()); } @@ -72,7 +73,7 @@ void testClosed() throws Exception { void testCleanupOnEmpty() throws Exception { assertTrue(Files.exists(queueRoot)); - queue.offer(toBytes("hello")); + queue.offer(toBytes(HELLO)); queue.poll(); queue.close(); @@ -83,7 +84,7 @@ void testCleanupOnEmpty() throws Exception { void testCleanupOnNotEmpty() throws Exception { assertTrue(Files.exists(queueRoot)); - queue.offer(toBytes("hello")); + queue.offer(toBytes(HELLO)); queue.close(); assertFalse(Files.exists(queueRoot)); diff --git a/airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java b/airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java index 7688e63607fd..c59526d1407e 100644 --- a/airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java +++ b/airbyte-scheduler/client/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java @@ -58,6 +58,7 @@ class DefaultSynchronousSchedulerClientTest { private static final UUID WORKSPACE_ID = UUID.randomUUID(); private static final UUID UUID1 = UUID.randomUUID(); private static final UUID UUID2 = UUID.randomUUID(); + private static final String UNCHECKED = "unchecked"; private static final JsonNode CONFIGURATION = Jsons.jsonNode(ImmutableMap.builder() .put("username", "airbyte") .put("password", "abc") @@ -70,6 +71,7 @@ class DefaultSynchronousSchedulerClientTest { .withDestinationId(UUID1) .withDestinationDefinitionId(UUID2) .withConfiguration(CONFIGURATION); + private static final String SOURCE_DOCKER_IMAGE = "source-airbyte:1.2.3"; private TemporalClient temporalClient; private JobTracker jobTracker; @@ -99,7 +101,7 @@ private static JobMetadata createMetadata(final boolean succeeded) { @DisplayName("Test execute method.") class ExecuteSynchronousJob { - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @Test void testExecuteJobSuccess() { final UUID sourceDefinitionId = UUID.randomUUID(); @@ -107,7 +109,7 @@ void testExecuteJobSuccess() { final Function mapperFunction = output -> output; when(function.get()).thenReturn(new TemporalResponse<>("hello", createMetadata(true))); - final ConnectorJobReportingContext jobContext = new ConnectorJobReportingContext(UUID.randomUUID(), "source-airbyte:1.2.3"); + final ConnectorJobReportingContext jobContext = new ConnectorJobReportingContext(UUID.randomUUID(), SOURCE_DOCKER_IMAGE); final SynchronousResponse response = schedulerClient .execute(ConfigType.DISCOVER_SCHEMA, jobContext, sourceDefinitionId, function, mapperFunction, WORKSPACE_ID); @@ -124,7 +126,7 @@ void testExecuteJobSuccess() { verifyNoInteractions(jobErrorReporter); } - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @Test void testExecuteMappedOutput() { final UUID sourceDefinitionId = UUID.randomUUID(); @@ -132,7 +134,7 @@ void testExecuteMappedOutput() { final Function mapperFunction = Object::toString; when(function.get()).thenReturn(new TemporalResponse<>(42, createMetadata(true))); - final ConnectorJobReportingContext jobContext = new ConnectorJobReportingContext(UUID.randomUUID(), "source-airbyte:1.2.3"); + final ConnectorJobReportingContext jobContext = new ConnectorJobReportingContext(UUID.randomUUID(), SOURCE_DOCKER_IMAGE); final SynchronousResponse response = schedulerClient .execute(ConfigType.DISCOVER_SCHEMA, jobContext, sourceDefinitionId, function, mapperFunction, WORKSPACE_ID); @@ -145,7 +147,7 @@ void testExecuteMappedOutput() { assertEquals(LOG_PATH, response.getMetadata().getLogPath()); } - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @Test void testExecuteJobFailure() { final UUID sourceDefinitionId = UUID.randomUUID(); @@ -153,7 +155,7 @@ void testExecuteJobFailure() { final Function mapperFunction = output -> output; when(function.get()).thenReturn(new TemporalResponse<>(null, createMetadata(false))); - final ConnectorJobReportingContext jobContext = new ConnectorJobReportingContext(UUID.randomUUID(), "source-airbyte:1.2.3"); + final ConnectorJobReportingContext jobContext = new ConnectorJobReportingContext(UUID.randomUUID(), SOURCE_DOCKER_IMAGE); final SynchronousResponse response = schedulerClient .execute(ConfigType.DISCOVER_SCHEMA, jobContext, sourceDefinitionId, function, mapperFunction, WORKSPACE_ID); @@ -170,7 +172,7 @@ void testExecuteJobFailure() { verifyNoInteractions(jobErrorReporter); } - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @Test void testExecuteRuntimeException() { final UUID sourceDefinitionId = UUID.randomUUID(); @@ -178,7 +180,7 @@ void testExecuteRuntimeException() { final Function mapperFunction = output -> output; when(function.get()).thenThrow(new RuntimeException()); - final ConnectorJobReportingContext jobContext = new ConnectorJobReportingContext(UUID.randomUUID(), "source-airbyte:1.2.3"); + final ConnectorJobReportingContext jobContext = new ConnectorJobReportingContext(UUID.randomUUID(), SOURCE_DOCKER_IMAGE); assertThrows( RuntimeException.class, () -> schedulerClient.execute(ConfigType.DISCOVER_SCHEMA, jobContext, sourceDefinitionId, function, diff --git a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java index 94de758be91c..ec899dbb3053 100644 --- a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java +++ b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java @@ -79,6 +79,11 @@ public class DefaultJobPersistence implements JobPersistence { private static final Set SYSTEM_SCHEMA = Set .of("pg_toast", "information_schema", "pg_catalog", "import_backup", "pg_internal", "catalog_history"); + public static final String ATTEMPT_NUMBER = "attempt_number"; + private static final String JOB_ID = "job_id"; + private static final String WHERE = "WHERE "; + private static final String AND = " AND "; + private static final String SCOPE_CLAUSE = "scope = ? AND "; protected static final String DEFAULT_SCHEMA = "public"; private static final String BACKUP_SCHEMA = "import_backup"; @@ -233,7 +238,7 @@ public int createAttempt(final long jobId, final Path logPath) throws IOExceptio now) .stream() .findFirst() - .map(r -> r.get("attempt_number", Integer.class)) + .map(r -> r.get(ATTEMPT_NUMBER, Integer.class)) .orElseThrow(() -> new RuntimeException("This should not happen")); }); @@ -340,7 +345,7 @@ public List listJobs(final ConfigType configType, final String configId, fi @Override public List listJobs(final Set configTypes, final String configId, final int pagesize, final int offset) throws IOException { return jobDatabase.query(ctx -> getJobsFromResult(ctx.fetch( - BASE_JOB_SELECT_AND_JOIN + "WHERE " + + BASE_JOB_SELECT_AND_JOIN + WHERE + "CAST(config_type AS VARCHAR) in " + Sqls.toSqlInFragment(configTypes) + " " + "AND scope = ? " + ORDER_BY_JOB_TIME_ATTEMPT_TIME + @@ -356,8 +361,8 @@ public List listJobsWithStatus(final JobStatus status) throws IOException { @Override public List listJobsWithStatus(final Set configTypes, final JobStatus status) throws IOException { return jobDatabase.query(ctx -> getJobsFromResult(ctx - .fetch(BASE_JOB_SELECT_AND_JOIN + "WHERE " + - "CAST(config_type AS VARCHAR) IN " + Sqls.toSqlInFragment(configTypes) + " AND " + + .fetch(BASE_JOB_SELECT_AND_JOIN + WHERE + + "CAST(config_type AS VARCHAR) IN " + Sqls.toSqlInFragment(configTypes) + AND + "CAST(jobs.status AS VARCHAR) = ? " + ORDER_BY_JOB_TIME_ATTEMPT_TIME, Sqls.toSqlName(status)))); @@ -372,9 +377,9 @@ public List listJobsWithStatus(final ConfigType configType, final JobStatus public List listJobsForConnectionWithStatuses(final UUID connectionId, final Set configTypes, final Set statuses) throws IOException { return jobDatabase.query(ctx -> getJobsFromResult(ctx - .fetch(BASE_JOB_SELECT_AND_JOIN + "WHERE " + - "scope = ? AND " + - "config_type IN " + Sqls.toSqlInFragment(configTypes) + " AND " + + .fetch(BASE_JOB_SELECT_AND_JOIN + WHERE + + SCOPE_CLAUSE + + "config_type IN " + Sqls.toSqlInFragment(configTypes) + AND + "jobs.status IN " + Sqls.toSqlInFragment(statuses) + " " + ORDER_BY_JOB_TIME_ATTEMPT_TIME, connectionId.toString()))); @@ -389,9 +394,9 @@ public List listJobStatusAndTimestampWithConnection(f final String JobStatusSelect = "SELECT id, status, created_at, updated_at FROM jobs "; return jobDatabase.query(ctx -> ctx - .fetch(JobStatusSelect + "WHERE " + - "scope = ? AND " + - "CAST(config_type AS VARCHAR) in " + Sqls.toSqlInFragment(configTypes) + " AND " + + .fetch(JobStatusSelect + WHERE + + SCOPE_CLAUSE + + "CAST(config_type AS VARCHAR) in " + Sqls.toSqlInFragment(configTypes) + AND + "created_at >= ? ORDER BY created_at DESC", connectionId.toString(), timeConvertedIntoLocalDateTime)) .stream() .map(r -> new JobWithStatusAndTimestamp( @@ -405,31 +410,31 @@ public List listJobStatusAndTimestampWithConnection(f @Override public Optional getLastReplicationJob(final UUID connectionId) throws IOException { return jobDatabase.query(ctx -> ctx - .fetch(BASE_JOB_SELECT_AND_JOIN + "WHERE " + - "CAST(jobs.config_type AS VARCHAR) in " + Sqls.toSqlInFragment(Job.REPLICATION_TYPES) + " AND " + - "scope = ? AND " + + .fetch(BASE_JOB_SELECT_AND_JOIN + WHERE + + "CAST(jobs.config_type AS VARCHAR) in " + Sqls.toSqlInFragment(Job.REPLICATION_TYPES) + AND + + SCOPE_CLAUSE + "CAST(jobs.status AS VARCHAR) <> ? " + "ORDER BY jobs.created_at DESC LIMIT 1", connectionId.toString(), Sqls.toSqlName(JobStatus.CANCELLED)) .stream() .findFirst() - .flatMap(r -> getJobOptional(ctx, r.get("job_id", Long.class)))); + .flatMap(r -> getJobOptional(ctx, r.get(JOB_ID, Long.class)))); } @Override public Optional getFirstReplicationJob(final UUID connectionId) throws IOException { return jobDatabase.query(ctx -> ctx - .fetch(BASE_JOB_SELECT_AND_JOIN + "WHERE " + - "CAST(jobs.config_type AS VARCHAR) in " + Sqls.toSqlInFragment(Job.REPLICATION_TYPES) + " AND " + - "scope = ? AND " + + .fetch(BASE_JOB_SELECT_AND_JOIN + WHERE + + "CAST(jobs.config_type AS VARCHAR) in " + Sqls.toSqlInFragment(Job.REPLICATION_TYPES) + AND + + SCOPE_CLAUSE + "CAST(jobs.status AS VARCHAR) <> ? " + "ORDER BY jobs.created_at ASC LIMIT 1", connectionId.toString(), Sqls.toSqlName(JobStatus.CANCELLED)) .stream() .findFirst() - .flatMap(r -> getJobOptional(ctx, r.get("job_id", Long.class)))); + .flatMap(r -> getJobOptional(ctx, r.get(JOB_ID, Long.class)))); } @Override @@ -439,20 +444,20 @@ public Optional getNextJob() throws IOException { // 2. job is excluded if another job of the same scope is already running // 3. job is excluded if another job of the same scope is already incomplete return jobDatabase.query(ctx -> ctx - .fetch(BASE_JOB_SELECT_AND_JOIN + "WHERE " + + .fetch(BASE_JOB_SELECT_AND_JOIN + WHERE + "CAST(jobs.status AS VARCHAR) = 'pending' AND " + "jobs.scope NOT IN ( SELECT scope FROM jobs WHERE status = 'running' OR status = 'incomplete' ) " + "ORDER BY jobs.created_at ASC LIMIT 1") .stream() .findFirst() - .flatMap(r -> getJobOptional(ctx, r.get("job_id", Long.class)))); + .flatMap(r -> getJobOptional(ctx, r.get(JOB_ID, Long.class)))); } @Override public List listJobs(final ConfigType configType, final Instant attemptEndedAtTimestamp) throws IOException { final LocalDateTime timeConvertedIntoLocalDateTime = LocalDateTime.ofInstant(attemptEndedAtTimestamp, ZoneOffset.UTC); return jobDatabase.query(ctx -> getJobsFromResult(ctx - .fetch(BASE_JOB_SELECT_AND_JOIN + "WHERE " + + .fetch(BASE_JOB_SELECT_AND_JOIN + WHERE + "CAST(config_type AS VARCHAR) = ? AND " + " attempts.ended_at > ? ORDER BY jobs.created_at ASC, attempts.created_at ASC", Sqls.toSqlName(configType), timeConvertedIntoLocalDateTime))); @@ -462,14 +467,14 @@ public List listJobs(final ConfigType configType, final Instant attemptEnde public List listAttemptsWithJobInfo(final ConfigType configType, final Instant attemptEndedAtTimestamp) throws IOException { final LocalDateTime timeConvertedIntoLocalDateTime = LocalDateTime.ofInstant(attemptEndedAtTimestamp, ZoneOffset.UTC); return jobDatabase.query(ctx -> getAttemptsWithJobsFromResult(ctx.fetch( - BASE_JOB_SELECT_AND_JOIN + "WHERE " + "CAST(config_type AS VARCHAR) = ? AND " + " attempts.ended_at > ? ORDER BY attempts.ended_at ASC", + BASE_JOB_SELECT_AND_JOIN + WHERE + "CAST(config_type AS VARCHAR) = ? AND " + " attempts.ended_at > ? ORDER BY attempts.ended_at ASC", Sqls.toSqlName(configType), timeConvertedIntoLocalDateTime))); } // Retrieves only Job information from the record, without any attempt info private static Job getJobFromRecord(final Record record) { - return new Job(record.get("job_id", Long.class), + return new Job(record.get(JOB_ID, Long.class), Enums.toEnum(record.get("config_type", String.class), ConfigType.class).orElseThrow(), record.get("scope", String.class), Jsons.deserialize(record.get("config", String.class), JobConfig.class), @@ -482,8 +487,8 @@ private static Job getJobFromRecord(final Record record) { private static Attempt getAttemptFromRecord(final Record record) { return new Attempt( - record.get("attempt_number", Long.class), - record.get("job_id", Long.class), + record.get(ATTEMPT_NUMBER, Long.class), + record.get(JOB_ID, Long.class), Path.of(record.get("log_path", String.class)), record.get("attempt_output", String.class) == null ? null : Jsons.deserialize(record.get("attempt_output", String.class), JobOutput.class), Enums.toEnum(record.get("attempt_status", String.class), AttemptStatus.class).orElseThrow(), @@ -499,7 +504,7 @@ private static Attempt getAttemptFromRecord(final Record record) { private static List getAttemptsWithJobsFromResult(final Result result) { return result .stream() - .filter(record -> record.getValue("attempt_number") != null) + .filter(record -> record.getValue(ATTEMPT_NUMBER) != null) .map(record -> new AttemptWithJobInfo(getAttemptFromRecord(record), getJobFromRecord(record))) .collect(Collectors.toList()); } @@ -509,11 +514,11 @@ private static List getJobsFromResult(final Result result) { final List jobs = new ArrayList(); Job currentJob = null; for (final Record entry : result) { - if (currentJob == null || currentJob.getId() != entry.get("job_id", Long.class)) { + if (currentJob == null || currentJob.getId() != entry.get(JOB_ID, Long.class)) { currentJob = getJobFromRecord(entry); jobs.add(currentJob); } - if (entry.getValue("attempt_number") != null) { + if (entry.getValue(ATTEMPT_NUMBER) != null) { currentJob.getAttempts().add(getAttemptFromRecord(entry)); } } diff --git a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/WorkspaceHelper.java b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/WorkspaceHelper.java index 520757baf92f..0e004110276e 100644 --- a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/WorkspaceHelper.java +++ b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/WorkspaceHelper.java @@ -29,6 +29,7 @@ // todo (cgardens) - this class is in an unintuitive module. it is weird that you need to import // scheduler:persistence in order to get workspace ids for configs (e.g. source). Our options are to // split this helper by database or put it in a new module. +@SuppressWarnings("PMD.AvoidCatchingThrowable") public class WorkspaceHelper { private static final Logger LOGGER = LoggerFactory.getLogger(WorkspaceHelper.class); diff --git a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/ResourceRequirementsUtilsTest.java b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/ResourceRequirementsUtilsTest.java index a2ab5350f208..9e679fab4c1b 100644 --- a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/ResourceRequirementsUtilsTest.java +++ b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/ResourceRequirementsUtilsTest.java @@ -15,6 +15,8 @@ class ResourceRequirementsUtilsTest { + private static final String FIVE_HUNDRED_MEM = "500Mi"; + @Test void testNoReqsSet() { final ResourceRequirements result = ResourceRequirementsUtils.getResourceRequirements( @@ -91,7 +93,8 @@ void testConnectionResourceRequirementsOverrideDefault() { final ActorDefinitionResourceRequirements definitionReqs = new ActorDefinitionResourceRequirements() .withDefault(definitionDefaultReqs) .withJobSpecific(List.of(jobTypeResourceLimit)); - final ResourceRequirements connectionResourceRequirements = new ResourceRequirements().withMemoryRequest("400Mi").withMemoryLimit("500Mi"); + final ResourceRequirements connectionResourceRequirements = + new ResourceRequirements().withMemoryRequest("400Mi").withMemoryLimit(FIVE_HUNDRED_MEM); final ResourceRequirements result = ResourceRequirementsUtils.getResourceRequirements( connectionResourceRequirements, @@ -103,21 +106,21 @@ void testConnectionResourceRequirementsOverrideDefault() { .withCpuRequest("2") .withCpuLimit("3") .withMemoryRequest("400Mi") - .withMemoryLimit("500Mi"); + .withMemoryLimit(FIVE_HUNDRED_MEM); assertEquals(expectedReqs, result); } @Test void testConnectionResourceRequirementsOverrideWorker() { final ResourceRequirements workerDefaultReqs = new ResourceRequirements().withCpuRequest("1").withCpuLimit("1"); - final ResourceRequirements connectionResourceRequirements = new ResourceRequirements().withCpuLimit("2").withMemoryLimit("500Mi"); + final ResourceRequirements connectionResourceRequirements = new ResourceRequirements().withCpuLimit("2").withMemoryLimit(FIVE_HUNDRED_MEM); final ResourceRequirements result = ResourceRequirementsUtils.getResourceRequirements(connectionResourceRequirements, workerDefaultReqs); final ResourceRequirements expectedReqs = new ResourceRequirements() .withCpuRequest("1") .withCpuLimit("2") - .withMemoryLimit("500Mi"); + .withMemoryLimit(FIVE_HUNDRED_MEM); assertEquals(expectedReqs, result); } diff --git a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/WebUrlHelperTest.java b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/WebUrlHelperTest.java index 66418954f338..29b386a518f2 100644 --- a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/WebUrlHelperTest.java +++ b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/WebUrlHelperTest.java @@ -12,11 +12,12 @@ class WebUrlHelperTest { private static final UUID WORKSPACE_ID = UUID.randomUUID(); private static final UUID CONNECTION_ID = UUID.randomUUID(); + private static final String LOCALHOST_8000 = "http://localhost:8000"; @Test void testGetBaseUrl() { - final WebUrlHelper webUrlHelper = new WebUrlHelper("http://localhost:8000"); - Assertions.assertEquals("http://localhost:8000", webUrlHelper.getBaseUrl()); + final WebUrlHelper webUrlHelper = new WebUrlHelper(LOCALHOST_8000); + Assertions.assertEquals(LOCALHOST_8000, webUrlHelper.getBaseUrl()); } @Test @@ -27,7 +28,7 @@ void testGetBaseUrlTrailingSlash() { @Test void testGetWorkspaceUrl() { - final WebUrlHelper webUrlHelper = new WebUrlHelper("http://localhost:8000"); + final WebUrlHelper webUrlHelper = new WebUrlHelper(LOCALHOST_8000); final String workspaceUrl = webUrlHelper.getWorkspaceUrl(WORKSPACE_ID); final String expectedUrl = String.format("http://localhost:8000/workspaces/%s", WORKSPACE_ID); Assertions.assertEquals(expectedUrl, workspaceUrl); @@ -35,7 +36,7 @@ void testGetWorkspaceUrl() { @Test void testGetConnectionUrl() { - final WebUrlHelper webUrlHelper = new WebUrlHelper("http://localhost:8000"); + final WebUrlHelper webUrlHelper = new WebUrlHelper(LOCALHOST_8000); final String connectionUrl = webUrlHelper.getConnectionUrl(WORKSPACE_ID, CONNECTION_ID); final String expectedUrl = String.format("http://localhost:8000/workspaces/%s/connections/%s", WORKSPACE_ID, CONNECTION_ID); Assertions.assertEquals(expectedUrl, connectionUrl); diff --git a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/JobErrorReporterTest.java b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/JobErrorReporterTest.java index eb8bf089b802..7ae415a5bad4 100644 --- a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/JobErrorReporterTest.java +++ b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/JobErrorReporterTest.java @@ -46,6 +46,20 @@ class JobErrorReporterTest { private static final String DESTINATION_DOCKER_REPOSITORY = "airbyte/destination-snowflake"; private static final String DESTINATION_DOCKER_IMAGE = "airbyte/destination-snowflake:1.2.3"; private static final StandardDestinationDefinition.ReleaseStage DESTINATION_RELEASE_STAGE = StandardDestinationDefinition.ReleaseStage.BETA; + private static final String FROM_TRACE_MESSAGE = "from_trace_message"; + private static final String JOB_ID_KEY = "job_id"; + private static final String WORKSPACE_ID_KEY = "workspace_id"; + private static final String DEPLOYMENT_MODE_KEY = "deployment_mode"; + private static final String AIRBYTE_VERSION_KEY = "airbyte_version"; + private static final String FAILURE_ORIGIN_KEY = "failure_origin"; + private static final String SOURCE = "source"; + private static final String FAILURE_TYPE_KEY = "failure_type"; + private static final String SYSTEM_ERROR = "system_error"; + private static final String CONNECTOR_DEFINITION_ID_KEY = "connector_definition_id"; + private static final String CONNECTOR_REPOSITORY_KEY = "connector_repository"; + private static final String CONNECTOR_NAME_KEY = "connector_name"; + private static final String CONNECTOR_RELEASE_STAGE_KEY = "connector_release_stage"; + private static final String CONNECTOR_COMMAND_KEY = "connector_command"; private ConfigRepository configRepository; private JobErrorReportingClient jobErrorReportingClient; @@ -65,12 +79,12 @@ void testReportSyncJobFailure() { final AttemptFailureSummary mFailureSummary = Mockito.mock(AttemptFailureSummary.class); final FailureReason sourceFailureReason = new FailureReason() - .withMetadata(new Metadata().withAdditionalProperty("from_trace_message", true)) + .withMetadata(new Metadata().withAdditionalProperty(FROM_TRACE_MESSAGE, true)) .withFailureOrigin(FailureOrigin.SOURCE) .withFailureType(FailureType.SYSTEM_ERROR); final FailureReason destinationFailureReason = new FailureReason() - .withMetadata(new Metadata().withAdditionalProperty("from_trace_message", true)) + .withMetadata(new Metadata().withAdditionalProperty(FROM_TRACE_MESSAGE, true)) .withFailureOrigin(FailureOrigin.DESTINATION) .withFailureType(FailureType.SYSTEM_ERROR); @@ -109,32 +123,32 @@ void testReportSyncJobFailure() { jobErrorReporter.reportSyncJobFailure(CONNECTION_ID, mFailureSummary, jobReportingContext); final Map expectedSourceMetadata = Map.ofEntries( - Map.entry("job_id", String.valueOf(syncJobId)), - Map.entry("workspace_id", WORKSPACE_ID.toString()), + Map.entry(JOB_ID_KEY, String.valueOf(syncJobId)), + Map.entry(WORKSPACE_ID_KEY, WORKSPACE_ID.toString()), Map.entry("connection_id", CONNECTION_ID.toString()), Map.entry("connection_url", CONNECTION_URL), - Map.entry("deployment_mode", DEPLOYMENT_MODE.name()), - Map.entry("airbyte_version", AIRBYTE_VERSION), - Map.entry("failure_origin", "source"), - Map.entry("failure_type", "system_error"), - Map.entry("connector_definition_id", SOURCE_DEFINITION_ID.toString()), - Map.entry("connector_repository", SOURCE_DOCKER_REPOSITORY), - Map.entry("connector_name", SOURCE_DEFINITION_NAME), - Map.entry("connector_release_stage", SOURCE_RELEASE_STAGE.toString())); + Map.entry(DEPLOYMENT_MODE_KEY, DEPLOYMENT_MODE.name()), + Map.entry(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION), + Map.entry(FAILURE_ORIGIN_KEY, SOURCE), + Map.entry(FAILURE_TYPE_KEY, SYSTEM_ERROR), + Map.entry(CONNECTOR_DEFINITION_ID_KEY, SOURCE_DEFINITION_ID.toString()), + Map.entry(CONNECTOR_REPOSITORY_KEY, SOURCE_DOCKER_REPOSITORY), + Map.entry(CONNECTOR_NAME_KEY, SOURCE_DEFINITION_NAME), + Map.entry(CONNECTOR_RELEASE_STAGE_KEY, SOURCE_RELEASE_STAGE.toString())); final Map expectedDestinationMetadata = Map.ofEntries( - Map.entry("job_id", String.valueOf(syncJobId)), - Map.entry("workspace_id", WORKSPACE_ID.toString()), + Map.entry(JOB_ID_KEY, String.valueOf(syncJobId)), + Map.entry(WORKSPACE_ID_KEY, WORKSPACE_ID.toString()), Map.entry("connection_id", CONNECTION_ID.toString()), Map.entry("connection_url", CONNECTION_URL), - Map.entry("deployment_mode", DEPLOYMENT_MODE.name()), - Map.entry("airbyte_version", AIRBYTE_VERSION), - Map.entry("failure_origin", "destination"), - Map.entry("failure_type", "system_error"), - Map.entry("connector_definition_id", DESTINATION_DEFINITION_ID.toString()), - Map.entry("connector_repository", DESTINATION_DOCKER_REPOSITORY), - Map.entry("connector_name", DESTINATION_DEFINITION_NAME), - Map.entry("connector_release_stage", DESTINATION_RELEASE_STAGE.toString())); + Map.entry(DEPLOYMENT_MODE_KEY, DEPLOYMENT_MODE.name()), + Map.entry(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION), + Map.entry(FAILURE_ORIGIN_KEY, "destination"), + Map.entry(FAILURE_TYPE_KEY, SYSTEM_ERROR), + Map.entry(CONNECTOR_DEFINITION_ID_KEY, DESTINATION_DEFINITION_ID.toString()), + Map.entry(CONNECTOR_REPOSITORY_KEY, DESTINATION_DOCKER_REPOSITORY), + Map.entry(CONNECTOR_NAME_KEY, DESTINATION_DEFINITION_NAME), + Map.entry(CONNECTOR_RELEASE_STAGE_KEY, DESTINATION_RELEASE_STAGE.toString())); Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, sourceFailureReason, SOURCE_DOCKER_IMAGE, expectedSourceMetadata); Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, destinationFailureReason, DESTINATION_DOCKER_IMAGE, @@ -148,7 +162,7 @@ void testReportSyncJobFailureDoesNotThrow() { final SyncJobReportingContext jobContext = new SyncJobReportingContext(1L, SOURCE_DOCKER_IMAGE, DESTINATION_DOCKER_IMAGE); final FailureReason sourceFailureReason = new FailureReason() - .withMetadata(new Metadata().withAdditionalProperty("from_trace_message", true)) + .withMetadata(new Metadata().withAdditionalProperty(FROM_TRACE_MESSAGE, true)) .withFailureOrigin(FailureOrigin.SOURCE) .withFailureType(FailureType.SYSTEM_ERROR); @@ -178,7 +192,7 @@ void testReportSyncJobFailureDoesNotThrow() { @Test void testReportSourceCheckJobFailure() throws JsonValidationException, ConfigNotFoundException, IOException { final FailureReason failureReason = new FailureReason() - .withMetadata(new Metadata().withAdditionalProperty("from_trace_message", true)) + .withMetadata(new Metadata().withAdditionalProperty(FROM_TRACE_MESSAGE, true)) .withFailureOrigin(FailureOrigin.SOURCE) .withFailureType(FailureType.SYSTEM_ERROR); @@ -198,17 +212,17 @@ void testReportSourceCheckJobFailure() throws JsonValidationException, ConfigNot jobErrorReporter.reportSourceCheckJobFailure(SOURCE_DEFINITION_ID, WORKSPACE_ID, failureReason, jobContext); final Map expectedMetadata = Map.ofEntries( - Map.entry("job_id", JOB_ID.toString()), - Map.entry("workspace_id", WORKSPACE_ID.toString()), - Map.entry("deployment_mode", DEPLOYMENT_MODE.name()), - Map.entry("airbyte_version", AIRBYTE_VERSION), - Map.entry("failure_origin", "source"), - Map.entry("failure_type", "system_error"), - Map.entry("connector_definition_id", SOURCE_DEFINITION_ID.toString()), - Map.entry("connector_repository", SOURCE_DOCKER_REPOSITORY), - Map.entry("connector_name", SOURCE_DEFINITION_NAME), - Map.entry("connector_release_stage", SOURCE_RELEASE_STAGE.toString()), - Map.entry("connector_command", "check")); + Map.entry(JOB_ID_KEY, JOB_ID.toString()), + Map.entry(WORKSPACE_ID_KEY, WORKSPACE_ID.toString()), + Map.entry(DEPLOYMENT_MODE_KEY, DEPLOYMENT_MODE.name()), + Map.entry(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION), + Map.entry(FAILURE_ORIGIN_KEY, SOURCE), + Map.entry(FAILURE_TYPE_KEY, SYSTEM_ERROR), + Map.entry(CONNECTOR_DEFINITION_ID_KEY, SOURCE_DEFINITION_ID.toString()), + Map.entry(CONNECTOR_REPOSITORY_KEY, SOURCE_DOCKER_REPOSITORY), + Map.entry(CONNECTOR_NAME_KEY, SOURCE_DEFINITION_NAME), + Map.entry(CONNECTOR_RELEASE_STAGE_KEY, SOURCE_RELEASE_STAGE.toString()), + Map.entry(CONNECTOR_COMMAND_KEY, "check")); Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); @@ -217,7 +231,7 @@ void testReportSourceCheckJobFailure() throws JsonValidationException, ConfigNot @Test void testReportDestinationCheckJobFailure() throws JsonValidationException, ConfigNotFoundException, IOException { final FailureReason failureReason = new FailureReason() - .withMetadata(new Metadata().withAdditionalProperty("from_trace_message", true)) + .withMetadata(new Metadata().withAdditionalProperty(FROM_TRACE_MESSAGE, true)) .withFailureOrigin(FailureOrigin.DESTINATION) .withFailureType(FailureType.SYSTEM_ERROR); @@ -237,17 +251,17 @@ void testReportDestinationCheckJobFailure() throws JsonValidationException, Conf jobErrorReporter.reportDestinationCheckJobFailure(DESTINATION_DEFINITION_ID, WORKSPACE_ID, failureReason, jobContext); final Map expectedMetadata = Map.ofEntries( - Map.entry("job_id", JOB_ID.toString()), - Map.entry("workspace_id", WORKSPACE_ID.toString()), - Map.entry("deployment_mode", DEPLOYMENT_MODE.name()), - Map.entry("airbyte_version", AIRBYTE_VERSION), - Map.entry("failure_origin", "destination"), - Map.entry("failure_type", "system_error"), - Map.entry("connector_definition_id", DESTINATION_DEFINITION_ID.toString()), - Map.entry("connector_repository", DESTINATION_DOCKER_REPOSITORY), - Map.entry("connector_name", DESTINATION_DEFINITION_NAME), - Map.entry("connector_release_stage", DESTINATION_RELEASE_STAGE.toString()), - Map.entry("connector_command", "check")); + Map.entry(JOB_ID_KEY, JOB_ID.toString()), + Map.entry(WORKSPACE_ID_KEY, WORKSPACE_ID.toString()), + Map.entry(DEPLOYMENT_MODE_KEY, DEPLOYMENT_MODE.name()), + Map.entry(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION), + Map.entry(FAILURE_ORIGIN_KEY, "destination"), + Map.entry(FAILURE_TYPE_KEY, SYSTEM_ERROR), + Map.entry(CONNECTOR_DEFINITION_ID_KEY, DESTINATION_DEFINITION_ID.toString()), + Map.entry(CONNECTOR_REPOSITORY_KEY, DESTINATION_DOCKER_REPOSITORY), + Map.entry(CONNECTOR_NAME_KEY, DESTINATION_DEFINITION_NAME), + Map.entry(CONNECTOR_RELEASE_STAGE_KEY, DESTINATION_RELEASE_STAGE.toString()), + Map.entry(CONNECTOR_COMMAND_KEY, "check")); Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, failureReason, DESTINATION_DOCKER_IMAGE, expectedMetadata); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); @@ -256,7 +270,7 @@ void testReportDestinationCheckJobFailure() throws JsonValidationException, Conf @Test void testReportDiscoverJobFailure() throws JsonValidationException, ConfigNotFoundException, IOException { final FailureReason failureReason = new FailureReason() - .withMetadata(new Metadata().withAdditionalProperty("from_trace_message", true)) + .withMetadata(new Metadata().withAdditionalProperty(FROM_TRACE_MESSAGE, true)) .withFailureOrigin(FailureOrigin.SOURCE) .withFailureType(FailureType.SYSTEM_ERROR); @@ -276,17 +290,17 @@ void testReportDiscoverJobFailure() throws JsonValidationException, ConfigNotFou jobErrorReporter.reportDiscoverJobFailure(SOURCE_DEFINITION_ID, WORKSPACE_ID, failureReason, jobContext); final Map expectedMetadata = Map.ofEntries( - Map.entry("job_id", JOB_ID.toString()), - Map.entry("workspace_id", WORKSPACE_ID.toString()), - Map.entry("deployment_mode", DEPLOYMENT_MODE.name()), - Map.entry("airbyte_version", AIRBYTE_VERSION), - Map.entry("failure_origin", "source"), - Map.entry("failure_type", "system_error"), - Map.entry("connector_definition_id", SOURCE_DEFINITION_ID.toString()), - Map.entry("connector_repository", SOURCE_DOCKER_REPOSITORY), - Map.entry("connector_name", SOURCE_DEFINITION_NAME), - Map.entry("connector_release_stage", SOURCE_RELEASE_STAGE.toString()), - Map.entry("connector_command", "discover")); + Map.entry(JOB_ID_KEY, JOB_ID.toString()), + Map.entry(WORKSPACE_ID_KEY, WORKSPACE_ID.toString()), + Map.entry(DEPLOYMENT_MODE_KEY, DEPLOYMENT_MODE.name()), + Map.entry(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION), + Map.entry(FAILURE_ORIGIN_KEY, SOURCE), + Map.entry(FAILURE_TYPE_KEY, SYSTEM_ERROR), + Map.entry(CONNECTOR_DEFINITION_ID_KEY, SOURCE_DEFINITION_ID.toString()), + Map.entry(CONNECTOR_REPOSITORY_KEY, SOURCE_DOCKER_REPOSITORY), + Map.entry(CONNECTOR_NAME_KEY, SOURCE_DEFINITION_NAME), + Map.entry(CONNECTOR_RELEASE_STAGE_KEY, SOURCE_RELEASE_STAGE.toString()), + Map.entry(CONNECTOR_COMMAND_KEY, "discover")); Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); @@ -295,7 +309,7 @@ void testReportDiscoverJobFailure() throws JsonValidationException, ConfigNotFou @Test void testReportSpecJobFailure() { final FailureReason failureReason = new FailureReason() - .withMetadata(new Metadata().withAdditionalProperty("from_trace_message", true)) + .withMetadata(new Metadata().withAdditionalProperty(FROM_TRACE_MESSAGE, true)) .withFailureOrigin(FailureOrigin.SOURCE) .withFailureType(FailureType.SYSTEM_ERROR); @@ -304,13 +318,13 @@ void testReportSpecJobFailure() { jobErrorReporter.reportSpecJobFailure(failureReason, jobContext); final Map expectedMetadata = Map.ofEntries( - Map.entry("job_id", JOB_ID.toString()), - Map.entry("deployment_mode", DEPLOYMENT_MODE.name()), - Map.entry("airbyte_version", AIRBYTE_VERSION), - Map.entry("failure_origin", "source"), - Map.entry("failure_type", "system_error"), - Map.entry("connector_repository", SOURCE_DOCKER_REPOSITORY), - Map.entry("connector_command", "spec")); + Map.entry(JOB_ID_KEY, JOB_ID.toString()), + Map.entry(DEPLOYMENT_MODE_KEY, DEPLOYMENT_MODE.name()), + Map.entry(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION), + Map.entry(FAILURE_ORIGIN_KEY, SOURCE), + Map.entry(FAILURE_TYPE_KEY, SYSTEM_ERROR), + Map.entry(CONNECTOR_REPOSITORY_KEY, SOURCE_DOCKER_REPOSITORY), + Map.entry(CONNECTOR_COMMAND_KEY, "spec")); Mockito.verify(jobErrorReportingClient).reportJobFailureReason(null, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); diff --git a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryExceptionHelperTest.java b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryExceptionHelperTest.java index 78c34b295d0d..81f8e6397bdf 100644 --- a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryExceptionHelperTest.java +++ b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryExceptionHelperTest.java @@ -15,6 +15,14 @@ class SentryExceptionHelperTest { + private static final String ERROR_PATH = "/airbyte/connector-errors/error.py"; + private static final String ABS_PATH = "abspath"; + private static final String LINE_NO = "lineno"; + private static final String FUNCTION = "function"; + private static final String CONTEXT_LINE = "context_line"; + private static final String FILENAME = "filename"; + private static final String MODULE = "module"; + final SentryExceptionHelper exceptionHelper = new SentryExceptionHelper(); @Test @@ -64,37 +72,37 @@ raise RuntimeError("My other error") from err assertExceptionContent(exceptionList.get(0), "requests.exceptions.HTTPError", "400 Client Error: Bad Request for url: https://airbyte.com", List.of( Map.of( - "abspath", "/airbyte/connector-errors/error.py", - "lineno", 31, - "function", "read_records", - "context_line", "failing_method()"), + ABS_PATH, ERROR_PATH, + LINE_NO, 31, + FUNCTION, "read_records", + CONTEXT_LINE, "failing_method()"), Map.of( - "abspath", "/airbyte/connector-errors/error.py", - "lineno", 36, - "function", "failing_method", - "context_line", "raise HTTPError(http_error_msg, response=self)"))); + ABS_PATH, ERROR_PATH, + LINE_NO, 36, + FUNCTION, "failing_method", + CONTEXT_LINE, "raise HTTPError(http_error_msg, response=self)"))); assertExceptionContent(exceptionList.get(1), "RuntimeError", "My other error", List.of( Map.of( - "abspath", "/airbyte/connector-errors/error.py", - "lineno", 39, - "function", "", - "context_line", "main()"), + ABS_PATH, ERROR_PATH, + LINE_NO, 39, + FUNCTION, "", + CONTEXT_LINE, "main()"), Map.of( - "abspath", "/airbyte/connector-errors/error.py", - "lineno", 13, - "function", "main", - "context_line", "sync_mode(\"incremental\")"), + ABS_PATH, ERROR_PATH, + LINE_NO, 13, + FUNCTION, "main", + CONTEXT_LINE, "sync_mode(\"incremental\")"), Map.of( - "abspath", "/airbyte/connector-errors/error.py", - "lineno", 17, - "function", "sync_mode", - "context_line", "incremental()"), + ABS_PATH, ERROR_PATH, + LINE_NO, 17, + FUNCTION, "sync_mode", + CONTEXT_LINE, "incremental()"), Map.of( - "abspath", "/airbyte/connector-errors/error.py", - "lineno", 33, - "function", "incremental", - "context_line", "raise RuntimeError(\"My other error\") from err"))); + ABS_PATH, ERROR_PATH, + LINE_NO, 33, + FUNCTION, "incremental", + CONTEXT_LINE, "raise RuntimeError(\"My other error\") from err"))); } @@ -115,10 +123,10 @@ raise RuntimeError() assertExceptionContent(exceptionList.get(0), "RuntimeError", null, List.of( Map.of( - "abspath", "/airbyte/connector-errors/error.py", - "lineno", 33, - "function", "incremental", - "context_line", "raise RuntimeError()"))); + ABS_PATH, ERROR_PATH, + LINE_NO, 33, + FUNCTION, "incremental", + CONTEXT_LINE, "raise RuntimeError()"))); } @Test @@ -155,17 +163,17 @@ raise _InactiveRpcError(state) assertExceptionContent(exceptionList.get(0), "grpc._channel._InactiveRpcError", expectedValue, List.of( Map.of( - "abspath", "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", - "lineno", 849, - "function", "_end_unary_response_blocking", - "context_line", "raise _InactiveRpcError(state)"))); + ABS_PATH, "/usr/local/lib/python3.9/site-packages/grpc/_channel.py", + LINE_NO, 849, + FUNCTION, "_end_unary_response_blocking", + CONTEXT_LINE, "raise _InactiveRpcError(state)"))); assertExceptionContent(exceptionList.get(1), "AttributeError", "'NoneType' object has no attribute 'from_call'", List.of( Map.of( - "abspath", "/usr/local/lib/python3.9/site-packages/google/api_core/exceptions.py", - "lineno", 553, - "function", "_parse_grpc_error_details", - "context_line", "status = rpc_status.from_call(rpc_exc)"))); + ABS_PATH, "/usr/local/lib/python3.9/site-packages/google/api_core/exceptions.py", + LINE_NO, 553, + FUNCTION, "_parse_grpc_error_details", + CONTEXT_LINE, "status = rpc_status.from_call(rpc_exc)"))); } @Test @@ -190,37 +198,37 @@ void testBuildSentryExceptionsJava() { assertExceptionContent(exceptionList.get(0), "java.lang.ArithmeticException", "/ by zero", List.of( Map.of( - "filename", "GradleWorkerMain.java", - "lineno", 74, - "module", "worker.org.gradle.process.internal.worker.GradleWorkerMain", - "function", "main"), + FILENAME, "GradleWorkerMain.java", + LINE_NO, 74, + MODULE, "worker.org.gradle.process.internal.worker.GradleWorkerMain", + FUNCTION, "main"), Map.of( - "module", "jdk.proxy2.$Proxy5", - "function", "stop"), + MODULE, "jdk.proxy2.$Proxy5", + FUNCTION, "stop"), Map.of( - "filename", "ThrowableCollector.java", - "lineno", 73, - "module", "org.junit.platform.engine.support.hierarchical.ThrowableCollector", - "function", "execute"), + FILENAME, "ThrowableCollector.java", + LINE_NO, 73, + MODULE, "org.junit.platform.engine.support.hierarchical.ThrowableCollector", + FUNCTION, "execute"), Map.of( - "filename", "NodeTestTask.java", - "lineno", 141, - "module", "org.junit.platform.engine.support.hierarchical.NodeTestTask", - "function", "lambda$executeRecursively$8"), + FILENAME, "NodeTestTask.java", + LINE_NO, 141, + MODULE, "org.junit.platform.engine.support.hierarchical.NodeTestTask", + FUNCTION, "lambda$executeRecursively$8"), Map.of( - "filename", "ExecutableInvoker.java", - "lineno", 115, - "module", "org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall", - "function", "lambda$ofVoidMethod$0"), + FILENAME, "ExecutableInvoker.java", + LINE_NO, 115, + MODULE, "org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall", + FUNCTION, "lambda$ofVoidMethod$0"), Map.of( "isNative", true, - "module", "jdk.internal.reflect.NativeMethodAccessorImpl", - "function", "invoke0"), + MODULE, "jdk.internal.reflect.NativeMethodAccessorImpl", + FUNCTION, "invoke0"), Map.of( - "filename", "AirbyteTraceMessageUtilityTest.java", - "lineno", 61, - "module", "io.airbyte.integrations.base.AirbyteTraceMessageUtilityTest", - "function", "testCorrectStacktraceFormat"))); + FILENAME, "AirbyteTraceMessageUtilityTest.java", + LINE_NO, 61, + MODULE, "io.airbyte.integrations.base.AirbyteTraceMessageUtilityTest", + FUNCTION, "testCorrectStacktraceFormat"))); } @Test @@ -252,33 +260,33 @@ void testBuildSentryExceptionsJavaChained() { "io.airbyte.workers.DefaultReplicationWorker$DestinationException: Destination process exited with non-zero exit code 1", List.of( Map.of( - "filename", "Thread.java", - "lineno", 833, - "module", "java.lang.Thread", - "function", "run"), + FILENAME, "Thread.java", + LINE_NO, 833, + MODULE, "java.lang.Thread", + FUNCTION, "run"), Map.of( - "filename", "ThreadPoolExecutor.java", - "lineno", 635, - "module", "java.util.concurrent.ThreadPoolExecutor$Worker", - "function", "run"), + FILENAME, "ThreadPoolExecutor.java", + LINE_NO, 635, + MODULE, "java.util.concurrent.ThreadPoolExecutor$Worker", + FUNCTION, "run"), Map.of( - "filename", "CompletableFuture.java", - "lineno", 315, - "module", "java.util.concurrent.CompletableFuture", - "function", "encodeThrowable"))); + FILENAME, "CompletableFuture.java", + LINE_NO, 315, + MODULE, "java.util.concurrent.CompletableFuture", + FUNCTION, "encodeThrowable"))); assertExceptionContent(exceptionList.get(1), "io.airbyte.workers.DefaultReplicationWorker$DestinationException", "Destination process exited with non-zero exit code 1", List.of( Map.of( - "filename", "CompletableFuture.java", - "lineno", 1804, - "module", "java.util.concurrent.CompletableFuture$AsyncRun", - "function", "run"), + FILENAME, "CompletableFuture.java", + LINE_NO, 1804, + MODULE, "java.util.concurrent.CompletableFuture$AsyncRun", + FUNCTION, "run"), Map.of( - "filename", "DefaultReplicationWorker.java", - "lineno", 397, - "module", "io.airbyte.workers.DefaultReplicationWorker", - "function", "lambda$getDestinationOutputRunnable$7"))); + FILENAME, "DefaultReplicationWorker.java", + LINE_NO, 397, + MODULE, "io.airbyte.workers.DefaultReplicationWorker", + FUNCTION, "lambda$getDestinationOutputRunnable$7"))); } @Test @@ -310,20 +318,20 @@ void testBuildSentryExceptionsJavaMultilineValue() { "message" : "Invalid Credentials" }""", List.of( Map.of( - "filename", "GoogleJsonResponseException.java", - "lineno", 146, - "module", "com.google.api.client.googleapis.json.GoogleJsonResponseException", - "function", "from"))); + FILENAME, "GoogleJsonResponseException.java", + LINE_NO, 146, + MODULE, "com.google.api.client.googleapis.json.GoogleJsonResponseException", + FUNCTION, "from"))); assertExceptionContent(exceptionList.get(1), "org.postgresql.util.PSQLException", """ ERROR: publication "airbyte_publication" does not exist Where: slot "airbyte_slot", output plugin "pgoutput", in the change callback, associated LSN 0/48029520""", List.of( Map.of( - "filename", "QueryExecutorImpl.java", - "lineno", 2675, - "module", "org.postgresql.core.v3.QueryExecutorImpl", - "function", "receiveErrorResponse"))); + FILENAME, "QueryExecutorImpl.java", + LINE_NO, 2675, + MODULE, "org.postgresql.core.v3.QueryExecutorImpl", + FUNCTION, "receiveErrorResponse"))); } private void assertExceptionContent(final SentryException exception, @@ -343,28 +351,28 @@ private void assertExceptionContent(final SentryException exception, final Map expectedFrame = frames.get(i); final SentryStackFrame sentryFrame = sentryFrames.get(i); - if (expectedFrame.containsKey("module")) { - Assertions.assertEquals(expectedFrame.get("module"), sentryFrame.getModule()); + if (expectedFrame.containsKey(MODULE)) { + Assertions.assertEquals(expectedFrame.get(MODULE), sentryFrame.getModule()); } - if (expectedFrame.containsKey("filename")) { - Assertions.assertEquals(expectedFrame.get("filename"), sentryFrame.getFilename()); + if (expectedFrame.containsKey(FILENAME)) { + Assertions.assertEquals(expectedFrame.get(FILENAME), sentryFrame.getFilename()); } - if (expectedFrame.containsKey("abspath")) { - Assertions.assertEquals(expectedFrame.get("abspath"), sentryFrame.getAbsPath()); + if (expectedFrame.containsKey(ABS_PATH)) { + Assertions.assertEquals(expectedFrame.get(ABS_PATH), sentryFrame.getAbsPath()); } - if (expectedFrame.containsKey("function")) { - Assertions.assertEquals(expectedFrame.get("function"), sentryFrame.getFunction()); + if (expectedFrame.containsKey(FUNCTION)) { + Assertions.assertEquals(expectedFrame.get(FUNCTION), sentryFrame.getFunction()); } - if (expectedFrame.containsKey("lineno")) { - Assertions.assertEquals(expectedFrame.get("lineno"), sentryFrame.getLineno()); + if (expectedFrame.containsKey(LINE_NO)) { + Assertions.assertEquals(expectedFrame.get(LINE_NO), sentryFrame.getLineno()); } - if (expectedFrame.containsKey("context_line")) { - Assertions.assertEquals(expectedFrame.get("context_line"), sentryFrame.getContextLine()); + if (expectedFrame.containsKey(CONTEXT_LINE)) { + Assertions.assertEquals(expectedFrame.get(CONTEXT_LINE), sentryFrame.getContextLine()); } if (expectedFrame.containsKey("isNative")) { diff --git a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryJobErrorReportingClientTest.java b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryJobErrorReportingClientTest.java index 492795fd4ef4..dfef08d2379a 100644 --- a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryJobErrorReportingClientTest.java +++ b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryJobErrorReportingClientTest.java @@ -37,6 +37,7 @@ class SentryJobErrorReportingClientTest { private static final UUID WORKSPACE_ID = UUID.randomUUID(); private static final String WORKSPACE_NAME = "My Workspace"; private static final String DOCKER_IMAGE = "airbyte/source-stripe:1.2.3"; + private static final String ERROR_MESSAGE = "RuntimeError: Something went wrong"; private final StandardWorkspace workspace = new StandardWorkspace().withWorkspaceId(WORKSPACE_ID).withName(WORKSPACE_NAME); private SentryJobErrorReportingClient sentryErrorReportingClient; @@ -80,7 +81,7 @@ void testReportJobFailureReason() { final FailureReason failureReason = new FailureReason() .withFailureOrigin(FailureOrigin.SOURCE) .withFailureType(FailureType.SYSTEM_ERROR) - .withInternalMessage("RuntimeError: Something went wrong"); + .withInternalMessage(ERROR_MESSAGE); final Map metadata = Map.of("some_metadata", "some_metadata_value"); sentryErrorReportingClient.reportJobFailureReason(workspace, failureReason, DOCKER_IMAGE, metadata); @@ -101,7 +102,7 @@ void testReportJobFailureReason() { final Message message = actualEvent.getMessage(); assertNotNull(message); - assertEquals("RuntimeError: Something went wrong", message.getFormatted()); + assertEquals(ERROR_MESSAGE, message.getFormatted()); } @Test @@ -111,7 +112,7 @@ void testReportJobFailureReasonWithNoWorkspace() { final FailureReason failureReason = new FailureReason() .withFailureOrigin(FailureOrigin.SOURCE) .withFailureType(FailureType.SYSTEM_ERROR) - .withInternalMessage("RuntimeError: Something went wrong"); + .withInternalMessage(ERROR_MESSAGE); sentryErrorReportingClient.reportJobFailureReason(null, failureReason, DOCKER_IMAGE, Map.of()); @@ -122,7 +123,7 @@ void testReportJobFailureReasonWithNoWorkspace() { final Message message = actualEvent.getMessage(); assertNotNull(message); - assertEquals("RuntimeError: Something went wrong", message.getFormatted()); + assertEquals(ERROR_MESSAGE, message.getFormatted()); } @Test @@ -138,7 +139,7 @@ void testReportJobFailureReasonWithStacktrace() { when(mockSentryExceptionHelper.buildSentryExceptions("Some valid stacktrace")).thenReturn(Optional.of(exceptions)); final FailureReason failureReason = new FailureReason() - .withInternalMessage("RuntimeError: Something went wrong") + .withInternalMessage(ERROR_MESSAGE) .withStacktrace("Some valid stacktrace"); sentryErrorReportingClient.reportJobFailureReason(workspace, failureReason, DOCKER_IMAGE, Map.of()); diff --git a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplierTest.java b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplierTest.java index 39f13dc22f18..12ed866eba41 100644 --- a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplierTest.java +++ b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_factory/OAuthConfigSupplierTest.java @@ -39,6 +39,10 @@ class OAuthConfigSupplierTest { static final String CREDENTIALS = "credentials"; static final String PROPERTIES = "properties"; + private static final String AUTH_TYPE = "auth_type"; + private static final String OAUTH = "oauth"; + private static final String API_SECRET = "api_secret"; + private ConfigRepository configRepository; private TrackingClient trackingClient; private OAuthConfigSupplier oAuthConfigSupplier; @@ -51,8 +55,8 @@ void setup() throws JsonValidationException, ConfigNotFoundException, IOExceptio oAuthConfigSupplier = new OAuthConfigSupplier(configRepository, trackingClient); sourceDefinitionId = UUID.randomUUID(); setupStandardDefinitionMock(createAdvancedAuth() - .withPredicateKey(List.of(CREDENTIALS, "auth_type")) - .withPredicateValue("oauth")); + .withPredicateKey(List.of(CREDENTIALS, AUTH_TYPE)) + .withPredicateValue(OAUTH)); } @Test @@ -67,8 +71,8 @@ void testNoOAuthInjectionBecauseEmptyParams() throws IOException { @Test void testNoOAuthInjectionBecauseMissingPredicateKey() throws IOException, JsonValidationException, ConfigNotFoundException { setupStandardDefinitionMock(createAdvancedAuth() - .withPredicateKey(List.of("some_random_fields", "auth_type")) - .withPredicateValue("oauth")); + .withPredicateKey(List.of("some_random_fields", AUTH_TYPE)) + .withPredicateValue(OAUTH)); final JsonNode config = generateJsonConfig(); final UUID workspaceId = UUID.randomUUID(); setupOAuthParamMocks(generateOAuthParameters()); @@ -80,7 +84,7 @@ void testNoOAuthInjectionBecauseMissingPredicateKey() throws IOException, JsonVa @Test void testNoOAuthInjectionBecauseWrongPredicateValue() throws IOException, JsonValidationException, ConfigNotFoundException { setupStandardDefinitionMock(createAdvancedAuth() - .withPredicateKey(List.of(CREDENTIALS, "auth_type")) + .withPredicateKey(List.of(CREDENTIALS, AUTH_TYPE)) .withPredicateValue("wrong_auth_type")); final JsonNode config = generateJsonConfig(); final UUID workspaceId = UUID.randomUUID(); @@ -120,7 +124,7 @@ void testOAuthInjectionWithoutPredicate() throws JsonValidationException, IOExce @Test void testOAuthInjectionWithoutPredicateValue() throws JsonValidationException, IOException, ConfigNotFoundException { setupStandardDefinitionMock(createAdvancedAuth() - .withPredicateKey(List.of(CREDENTIALS, "auth_type")) + .withPredicateKey(List.of(CREDENTIALS, AUTH_TYPE)) .withPredicateValue("")); final JsonNode config = generateJsonConfig(); final UUID workspaceId = UUID.randomUUID(); @@ -188,8 +192,8 @@ void testOAuthFullInjectionBecauseNoOAuthSpecNestedParameters() throws JsonValid final JsonNode expectedConfig = Jsons.jsonNode(Map.of( "fieldName", "fieldValue", CREDENTIALS, Map.of( - "api_secret", "123", - "auth_type", "oauth", + API_SECRET, "123", + AUTH_TYPE, OAUTH, API_CLIENT, ((Map) oauthParameters.get(CREDENTIALS)).get(API_CLIENT)))); assertEquals(expectedConfig, actualConfig); assertTracking(workspaceId); @@ -247,13 +251,13 @@ private static ObjectNode generateJsonConfig() { Map.of( "fieldName", "fieldValue", CREDENTIALS, Map.of( - "api_secret", "123", - "auth_type", "oauth"))); + API_SECRET, "123", + AUTH_TYPE, OAUTH))); } private static Map generateOAuthParameters() { return Map.of( - "api_secret", "mysecret", + API_SECRET, "mysecret", API_CLIENT, UUID.randomUUID().toString()); } @@ -266,8 +270,8 @@ private static JsonNode getExpectedNode(final String apiClient) { Map.of( "fieldName", "fieldValue", CREDENTIALS, Map.of( - "api_secret", "123", - "auth_type", "oauth", + API_SECRET, "123", + AUTH_TYPE, OAUTH, API_CLIENT, apiClient))); } diff --git a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_tracker/JobTrackerTest.java b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_tracker/JobTrackerTest.java index b00582ca14c0..77be10539181 100644 --- a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_tracker/JobTrackerTest.java +++ b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/job_tracker/JobTrackerTest.java @@ -79,6 +79,12 @@ class JobTrackerTest { private static final String DESTINATION_DEF_NAME = "bigquery"; private static final String CONNECTOR_REPOSITORY = "test/test"; private static final String CONNECTOR_VERSION = "test"; + private static final String JOB_TYPE = "job_type"; + private static final String JOB_ID_KEY = "job_id"; + private static final String ATTEMPT_ID = "attempt_id"; + private static final String METADATA = "metadata"; + private static final String SOME = "some"; + private static final long SYNC_START_TIME = 1000L; private static final long SYNC_END_TIME = 10000L; private static final long SYNC_DURATION = 9L; // in sync between end and start time @@ -164,9 +170,9 @@ void setup() { @Test void testTrackCheckConnectionSource() throws ConfigNotFoundException, IOException, JsonValidationException { final ImmutableMap metadata = ImmutableMap.builder() - .put("job_type", ConfigType.CHECK_CONNECTION_SOURCE) - .put("job_id", JOB_ID.toString()) - .put("attempt_id", 0) + .put(JOB_TYPE, ConfigType.CHECK_CONNECTION_SOURCE) + .put(JOB_ID_KEY, JOB_ID.toString()) + .put(ATTEMPT_ID, 0) .put("connector_source", SOURCE_DEF_NAME) .put("connector_source_definition_id", UUID1) .put("connector_source_docker_repository", CONNECTOR_REPOSITORY) @@ -194,9 +200,9 @@ void testTrackCheckConnectionSource() throws ConfigNotFoundException, IOExceptio @Test void testTrackCheckConnectionDestination() throws ConfigNotFoundException, IOException, JsonValidationException { final ImmutableMap metadata = ImmutableMap.builder() - .put("job_type", ConfigType.CHECK_CONNECTION_DESTINATION) - .put("job_id", JOB_ID.toString()) - .put("attempt_id", 0) + .put(JOB_TYPE, ConfigType.CHECK_CONNECTION_DESTINATION) + .put(JOB_ID_KEY, JOB_ID.toString()) + .put(ATTEMPT_ID, 0) .put("connector_destination", DESTINATION_DEF_NAME) .put("connector_destination_definition_id", UUID2) .put("connector_destination_docker_repository", CONNECTOR_REPOSITORY) @@ -224,9 +230,9 @@ void testTrackCheckConnectionDestination() throws ConfigNotFoundException, IOExc @Test void testTrackDiscover() throws ConfigNotFoundException, IOException, JsonValidationException { final ImmutableMap metadata = ImmutableMap.builder() - .put("job_type", ConfigType.DISCOVER_SCHEMA) - .put("job_id", JOB_ID.toString()) - .put("attempt_id", 0) + .put(JOB_TYPE, ConfigType.DISCOVER_SCHEMA) + .put(JOB_ID_KEY, JOB_ID.toString()) + .put(ATTEMPT_ID, 0) .put("connector_source", SOURCE_DEF_NAME) .put("connector_source_definition_id", UUID1) .put("connector_source_docker_repository", CONNECTOR_REPOSITORY) @@ -350,7 +356,7 @@ void testAsynchronousAttemptWithFailures(final ConfigType configType, final Map< linkedHashMap.put("failureType", "config_error"); linkedHashMap.put("internalMessage", "Internal config error error msg"); linkedHashMap.put("externalMessage", "Config error related msg"); - linkedHashMap.put("metadata", ImmutableMap.of("some", "metadata")); + linkedHashMap.put(METADATA, ImmutableMap.of(SOME, METADATA)); linkedHashMap.put("retryable", true); linkedHashMap.put("timestamp", 1010); final JsonNode configFailureJson = Jsons.jsonNode(linkedHashMap); @@ -360,7 +366,7 @@ void testAsynchronousAttemptWithFailures(final ConfigType configType, final Map< linkedHashMap1.put("failureType", "system_error"); linkedHashMap1.put("internalMessage", "Internal system error error msg"); linkedHashMap1.put("externalMessage", "System error related msg"); - linkedHashMap1.put("metadata", ImmutableMap.of("some", "metadata")); + linkedHashMap1.put(METADATA, ImmutableMap.of(SOME, METADATA)); linkedHashMap1.put("retryable", true); linkedHashMap1.put("timestamp", 1100); final JsonNode systemFailureJson = Jsons.jsonNode(linkedHashMap1); @@ -370,7 +376,7 @@ void testAsynchronousAttemptWithFailures(final ConfigType configType, final Map< linkedHashMap2.put("failureType", null); linkedHashMap2.put("internalMessage", "Internal unknown error error msg"); linkedHashMap2.put("externalMessage", "Unknown error related msg"); - linkedHashMap2.put("metadata", ImmutableMap.of("some", "metadata")); + linkedHashMap2.put(METADATA, ImmutableMap.of(SOME, METADATA)); linkedHashMap2.put("retryable", true); linkedHashMap2.put("timestamp", 1110); final JsonNode unknownFailureJson = Jsons.jsonNode(linkedHashMap2); @@ -497,7 +503,7 @@ private List getAttemptsWithFailuresMock() { .withFailureOrigin(FailureReason.FailureOrigin.SOURCE) .withFailureType(FailureReason.FailureType.CONFIG_ERROR) .withRetryable(true) - .withMetadata(new Metadata().withAdditionalProperty("some", "metadata")) + .withMetadata(new Metadata().withAdditionalProperty(SOME, METADATA)) .withExternalMessage("Config error related msg") .withInternalMessage("Internal config error error msg") .withStacktrace("Don't include stacktrace in call to track") @@ -511,14 +517,14 @@ private List getAttemptsWithFailuresMock() { .withFailureOrigin(FailureReason.FailureOrigin.REPLICATION) .withFailureType(FailureReason.FailureType.SYSTEM_ERROR) .withRetryable(true) - .withMetadata(new Metadata().withAdditionalProperty("some", "metadata")) + .withMetadata(new Metadata().withAdditionalProperty(SOME, METADATA)) .withExternalMessage("System error related msg") .withInternalMessage("Internal system error error msg") .withStacktrace("Don't include stacktrace in call to track") .withTimestamp(SYNC_START_TIME + 100); final FailureReason unknownFailureReason = new FailureReason() .withRetryable(true) - .withMetadata(new Metadata().withAdditionalProperty("some", "metadata")) + .withMetadata(new Metadata().withAdditionalProperty(SOME, METADATA)) .withExternalMessage("Unknown error related msg") .withInternalMessage("Internal unknown error error msg") .withStacktrace("Don't include stacktrace in call to track") @@ -541,9 +547,9 @@ private Job getJobWithFailuresMock(final ConfigType configType, final long jobId private ImmutableMap getJobMetadata(final ConfigType configType, final long jobId) { return ImmutableMap.builder() - .put("job_type", configType) - .put("job_id", String.valueOf(jobId)) - .put("attempt_id", 700) + .put(JOB_TYPE, configType) + .put(JOB_ID_KEY, String.valueOf(jobId)) + .put(ATTEMPT_ID, 700) .put("connection_id", CONNECTION_ID) .put("connector_source", SOURCE_DEF_NAME) .put("connector_source_definition_id", UUID1) diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 059b7777ea11..1a05d994faca 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -79,6 +79,7 @@ import org.slf4j.LoggerFactory; import org.slf4j.MDC; +@SuppressWarnings("PMD.AvoidCatchingThrowable") public class ServerApp implements ServerRunnable { private static final Logger LOGGER = LoggerFactory.getLogger(ServerApp.class); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java index 455101f17850..8949eab082f2 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java @@ -46,6 +46,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +@SuppressWarnings("PMD.AvoidCatchingNPE") public class DestinationDefinitionsHandler { private final ConfigRepository configRepository; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java index c4a5785362a5..39614ed1bf38 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java @@ -36,6 +36,7 @@ public class OAuthHandler { private static final Logger LOGGER = LoggerFactory.getLogger(OAuthHandler.class); + private static final String ERROR_MESSAGE = "failed while reporting usage."; private final ConfigRepository configRepository; private final OAuthImplementationFactory oAuthImplementationFactory; @@ -73,7 +74,7 @@ public OAuthConsentRead getSourceOAuthConsent(final SourceOauthConsentRequest so try { trackingClient.track(sourceDefinitionIdRequestBody.getWorkspaceId(), "Get Oauth Consent URL - Backend", metadata); } catch (final Exception e) { - LOGGER.error("failed while reporting usage.", e); + LOGGER.error(ERROR_MESSAGE, e); } return result; } @@ -102,7 +103,7 @@ public OAuthConsentRead getDestinationOAuthConsent(final DestinationOauthConsent try { trackingClient.track(destinationDefinitionIdRequestBody.getWorkspaceId(), "Get Oauth Consent URL - Backend", metadata); } catch (final Exception e) { - LOGGER.error("failed while reporting usage.", e); + LOGGER.error(ERROR_MESSAGE, e); } return result; } @@ -134,7 +135,7 @@ public Map completeSourceOAuth(final CompleteSourceOauthRequest try { trackingClient.track(oauthSourceRequestBody.getWorkspaceId(), "Complete OAuth Flow - Backend", metadata); } catch (final Exception e) { - LOGGER.error("failed while reporting usage.", e); + LOGGER.error(ERROR_MESSAGE, e); } return result; } @@ -166,7 +167,7 @@ public Map completeDestinationOAuth(final CompleteDestinationOAu try { trackingClient.track(oauthDestinationRequestBody.getWorkspaceId(), "Complete OAuth Flow - Backend", metadata); } catch (final Exception e) { - LOGGER.error("failed while reporting usage.", e); + LOGGER.error(ERROR_MESSAGE, e); } return result; } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java index 67ca7c9ea430..64d26270bebb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java @@ -47,6 +47,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +@SuppressWarnings("PMD.AvoidCatchingNPE") public class SourceDefinitionsHandler { private final ConfigRepository configRepository; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java index b5f2c6cb2940..cd914563d50e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java @@ -225,7 +225,8 @@ private String generateUniqueSlug(final String workspaceName) throws JsonValidat // database transaction, but that is not something we can do quickly. resolvedSlug = proposedSlug + "-" + RandomStringUtils.randomAlphabetic(8); isSlugUsed = configRepository.getWorkspaceBySlugOptional(resolvedSlug, true).isPresent(); - if (count++ > MAX_ATTEMPTS) { + count++; + if (count > MAX_ATTEMPTS) { throw new InternalServerKnownException(String.format("could not generate a valid slug after %s tries.", MAX_ATTEMPTS)); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java b/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java index 92bbf3b698fd..b9683acd7b6d 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java +++ b/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java @@ -22,6 +22,7 @@ /** * Convenience class for retrieving files checked into the Airbyte Github repo. */ +@SuppressWarnings("PMD.AvoidCatchingThrowable") public class AirbyteGithubStore { private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteGithubStore.class); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java index bcdadc52f8a6..65b5374793df 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java @@ -86,6 +86,18 @@ class ConnectionsHandlerTest { private TrackingClient trackingClient; private EventRunner eventRunner; + private static final String PRESTO_TO_HUDI = "presto to hudi"; + private static final String PRESTO_TO_HUDI_PREFIX = "presto_to_hudi"; + private static final String SOURCE_TEST = "source-test"; + private static final String DESTINATION_TEST = "destination-test"; + private static final String CURSOR1 = "cursor1"; + private static final String CURSOR2 = "cursor2"; + private static final String PK1 = "pk1"; + private static final String PK2 = "pk2"; + private static final String PK3 = "pk3"; + private static final String STREAM1 = "stream1"; + private static final String STREAM2 = "stream2"; + @SuppressWarnings("unchecked") @BeforeEach void setUp() throws IOException, JsonValidationException, ConfigNotFoundException { @@ -106,10 +118,10 @@ void setUp() throws IOException, JsonValidationException, ConfigNotFoundExceptio .withConfiguration(Jsons.jsonNode(Collections.singletonMap("apiKey", "123-abc"))); standardSync = new StandardSync() .withConnectionId(connectionId) - .withName("presto to hudi") + .withName(PRESTO_TO_HUDI) .withNamespaceDefinition(JobSyncConfig.NamespaceDefinitionType.SOURCE) .withNamespaceFormat(null) - .withPrefix("presto_to_hudi") + .withPrefix(PRESTO_TO_HUDI_PREFIX) .withStatus(StandardSync.Status.ACTIVE) .withCatalog(ConnectionHelpers.generateBasicConfiguredAirbyteCatalog()) .withSourceId(sourceId) @@ -164,10 +176,10 @@ void setUp() { void testCreateConnection() throws JsonValidationException, ConfigNotFoundException, IOException { when(uuidGenerator.get()).thenReturn(standardSync.getConnectionId()); final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() - .withName("source-test") + .withName(SOURCE_TEST) .withSourceDefinitionId(UUID.randomUUID()); final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() - .withName("destination-test") + .withName(DESTINATION_TEST) .withDestinationDefinitionId(UUID.randomUUID()); when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); when(configRepository.getSourceDefinitionFromConnection(standardSync.getConnectionId())).thenReturn(sourceDefinition); @@ -183,10 +195,10 @@ void testCreateConnection() throws JsonValidationException, ConfigNotFoundExcept .sourceId(standardSync.getSourceId()) .destinationId(standardSync.getDestinationId()) .operationIds(standardSync.getOperationIds()) - .name("presto to hudi") + .name(PRESTO_TO_HUDI) .namespaceDefinition(NamespaceDefinitionType.SOURCE) .namespaceFormat(null) - .prefix("presto_to_hudi") + .prefix(PRESTO_TO_HUDI_PREFIX) .status(ConnectionStatus.ACTIVE) .schedule(ConnectionHelpers.generateBasicConnectionSchedule()) .syncCatalog(catalog) @@ -245,10 +257,10 @@ void testCreateConnectionWithBadDefinitionIds() throws JsonValidationException, final UUID destinationIdBad = UUID.randomUUID(); final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() - .withName("source-test") + .withName(SOURCE_TEST) .withSourceDefinitionId(UUID.randomUUID()); final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() - .withName("destination-test") + .withName(DESTINATION_TEST) .withDestinationDefinitionId(UUID.randomUUID()); when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); when(configRepository.getSourceDefinitionFromConnection(standardSync.getConnectionId())).thenReturn(sourceDefinition); @@ -265,10 +277,10 @@ void testCreateConnectionWithBadDefinitionIds() throws JsonValidationException, .sourceId(sourceIdBad) .destinationId(standardSync.getDestinationId()) .operationIds(standardSync.getOperationIds()) - .name("presto to hudi") + .name(PRESTO_TO_HUDI) .namespaceDefinition(NamespaceDefinitionType.SOURCE) .namespaceFormat(null) - .prefix("presto_to_hudi") + .prefix(PRESTO_TO_HUDI_PREFIX) .status(ConnectionStatus.ACTIVE) .schedule(ConnectionHelpers.generateBasicConnectionSchedule()) .syncCatalog(catalog); @@ -279,10 +291,10 @@ void testCreateConnectionWithBadDefinitionIds() throws JsonValidationException, .sourceId(standardSync.getSourceId()) .destinationId(destinationIdBad) .operationIds(standardSync.getOperationIds()) - .name("presto to hudi") + .name(PRESTO_TO_HUDI) .namespaceDefinition(NamespaceDefinitionType.SOURCE) .namespaceFormat(null) - .prefix("presto_to_hudi") + .prefix(PRESTO_TO_HUDI_PREFIX) .status(ConnectionStatus.ACTIVE) .schedule(ConnectionHelpers.generateBasicConnectionSchedule()) .syncCatalog(catalog); @@ -321,10 +333,10 @@ void testUpdateConnection() throws JsonValidationException, ConfigNotFoundExcept final StandardSync updatedStandardSync = new StandardSync() .withConnectionId(standardSync.getConnectionId()) - .withName("presto to hudi") + .withName(PRESTO_TO_HUDI) .withNamespaceDefinition(io.airbyte.config.JobSyncConfig.NamespaceDefinitionType.SOURCE) .withNamespaceFormat(standardSync.getNamespaceFormat()) - .withPrefix("presto_to_hudi") + .withPrefix(PRESTO_TO_HUDI_PREFIX) .withSourceId(standardSync.getSourceId()) .withDestinationId(standardSync.getDestinationId()) .withOperationIds(standardSync.getOperationIds()) @@ -437,10 +449,10 @@ void testSearchConnections() throws JsonValidationException, ConfigNotFoundExcep .withResourceRequirements(ConnectionHelpers.TESTING_RESOURCE_REQUIREMENTS); final ConnectionRead connectionRead2 = ConnectionHelpers.connectionReadFromStandardSync(standardSync2); final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() - .withName("source-test") + .withName(SOURCE_TEST) .withSourceDefinitionId(UUID.randomUUID()); final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() - .withName("destination-test") + .withName(DESTINATION_TEST) .withDestinationDefinitionId(UUID.randomUUID()); when(configRepository.listStandardSyncs()) @@ -568,10 +580,10 @@ void failOnUnmatchedWorkspacesInCreate() throws JsonValidationException, ConfigN when(uuidGenerator.get()).thenReturn(standardSync.getConnectionId()); final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() - .withName("source-test") + .withName(SOURCE_TEST) .withSourceDefinitionId(UUID.randomUUID()); final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() - .withName("destination-test") + .withName(DESTINATION_TEST) .withDestinationDefinitionId(UUID.randomUUID()); when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); when(configRepository.getSourceDefinitionFromConnection(standardSync.getConnectionId())).thenReturn(sourceDefinition); @@ -583,10 +595,10 @@ void failOnUnmatchedWorkspacesInCreate() throws JsonValidationException, ConfigN .sourceId(standardSync.getSourceId()) .destinationId(standardSync.getDestinationId()) .operationIds(standardSync.getOperationIds()) - .name("presto to hudi") + .name(PRESTO_TO_HUDI) .namespaceDefinition(NamespaceDefinitionType.SOURCE) .namespaceFormat(null) - .prefix("presto_to_hudi") + .prefix(PRESTO_TO_HUDI_PREFIX) .status(ConnectionStatus.ACTIVE) .schedule(ConnectionHelpers.generateBasicConnectionSchedule()) .syncCatalog(catalog) @@ -630,27 +642,27 @@ void setUp() { @Test void testNoDiff() { final AirbyteStreamConfiguration streamConfiguration1 = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1")), + List.of(CURSOR1), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration2 = getStreamConfiguration( - List.of("cursor2"), - List.of(List.of("pk2")), + List.of(CURSOR2), + List.of(List.of(PK2)), SyncMode.FULL_REFRESH, DestinationSyncMode.OVERWRITE); final AirbyteCatalog catalog1 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1), + getStreamAndConfig(STREAM2, streamConfiguration2))); final AirbyteCatalog catalog2 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1), + getStreamAndConfig(STREAM2, streamConfiguration2))); assertTrue(connectionsHandler.getConfigurationDiff(catalog1, catalog2).isEmpty()); } @@ -658,26 +670,26 @@ void testNoDiff() { @Test void testNoDiffIfStreamAdded() { final AirbyteStreamConfiguration streamConfiguration1 = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1")), + List.of(CURSOR1), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration2 = getStreamConfiguration( - List.of("cursor2"), - List.of(List.of("pk2")), + List.of(CURSOR2), + List.of(List.of(PK2)), SyncMode.FULL_REFRESH, DestinationSyncMode.OVERWRITE); final AirbyteCatalog catalog1 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1))); + getStreamAndConfig(STREAM1, streamConfiguration1))); final AirbyteCatalog catalog2 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1), + getStreamAndConfig(STREAM2, streamConfiguration2))); assertTrue(connectionsHandler.getConfigurationDiff(catalog1, catalog2).isEmpty()); } @@ -685,106 +697,106 @@ void testNoDiffIfStreamAdded() { @Test void testCursorOrderDoesMatter() { final AirbyteStreamConfiguration streamConfiguration1 = getStreamConfiguration( - List.of("cursor1", "anotherCursor"), - List.of(List.of("pk1")), + List.of(CURSOR1, "anotherCursor"), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration1WithOtherCursorOrder = getStreamConfiguration( - List.of("anotherCursor", "cursor1"), - List.of(List.of("pk1")), + List.of("anotherCursor", CURSOR1), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration2 = getStreamConfiguration( - List.of("cursor2"), - List.of(List.of("pk2")), + List.of(CURSOR2), + List.of(List.of(PK2)), SyncMode.FULL_REFRESH, DestinationSyncMode.OVERWRITE); final AirbyteCatalog catalog1 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1), + getStreamAndConfig(STREAM2, streamConfiguration2))); final AirbyteCatalog catalog2 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1WithOtherCursorOrder), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1WithOtherCursorOrder), + getStreamAndConfig(STREAM2, streamConfiguration2))); final Set changedSd = connectionsHandler.getConfigurationDiff(catalog1, catalog2); assertFalse(changedSd.isEmpty()); assertEquals(1, changedSd.size()); - assertEquals(Set.of(new StreamDescriptor().name("stream1")), changedSd); + assertEquals(Set.of(new StreamDescriptor().name(STREAM1)), changedSd); } @Test void testPkOrderDoesntMatter() { final AirbyteStreamConfiguration streamConfiguration1 = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1", "pk3")), + List.of(CURSOR1), + List.of(List.of(PK1, PK3)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration1WithOtherPkOrder = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk3", "pk1")), + List.of(CURSOR1), + List.of(List.of(PK3, PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration2 = getStreamConfiguration( - List.of("cursor2"), - List.of(List.of("pk2"), List.of("pk3")), + List.of(CURSOR2), + List.of(List.of(PK2), List.of(PK3)), SyncMode.FULL_REFRESH, DestinationSyncMode.OVERWRITE); final AirbyteStreamConfiguration streamConfiguration2WithOtherPkOrder = getStreamConfiguration( - List.of("cursor2"), - List.of(List.of("pk3"), List.of("pk2")), + List.of(CURSOR2), + List.of(List.of(PK3), List.of(PK2)), SyncMode.FULL_REFRESH, DestinationSyncMode.OVERWRITE); final AirbyteCatalog catalog1 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1), + getStreamAndConfig(STREAM2, streamConfiguration2))); final AirbyteCatalog catalog2 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1WithOtherPkOrder), - getStreamAndConfig("stream2", streamConfiguration2WithOtherPkOrder))); + getStreamAndConfig(STREAM1, streamConfiguration1WithOtherPkOrder), + getStreamAndConfig(STREAM2, streamConfiguration2WithOtherPkOrder))); final Set changedSd = connectionsHandler.getConfigurationDiff(catalog1, catalog2); assertFalse(changedSd.isEmpty()); assertEquals(1, changedSd.size()); - assertEquals(Set.of(new StreamDescriptor().name("stream1")), changedSd); + assertEquals(Set.of(new StreamDescriptor().name(STREAM1)), changedSd); } @Test void testNoDiffIfStreamRemove() { final AirbyteStreamConfiguration streamConfiguration1 = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1")), + List.of(CURSOR1), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration2 = getStreamConfiguration( - List.of("cursor2"), - List.of(List.of("pk2")), + List.of(CURSOR2), + List.of(List.of(PK2)), SyncMode.FULL_REFRESH, DestinationSyncMode.OVERWRITE); final AirbyteCatalog catalog1 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1), + getStreamAndConfig(STREAM2, streamConfiguration2))); final AirbyteCatalog catalog2 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1))); + getStreamAndConfig(STREAM1, streamConfiguration1))); assertTrue(connectionsHandler.getConfigurationDiff(catalog1, catalog2).isEmpty()); } @@ -792,156 +804,156 @@ void testNoDiffIfStreamRemove() { @Test void testDiffDifferentCursor() { final AirbyteStreamConfiguration streamConfiguration1 = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1")), + List.of(CURSOR1), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration1CursorDiff = getStreamConfiguration( - List.of("cursor1", "anotherCursor"), - List.of(List.of("pk1")), + List.of(CURSOR1, "anotherCursor"), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration2 = getStreamConfiguration( - List.of("cursor2"), - List.of(List.of("pk2")), + List.of(CURSOR2), + List.of(List.of(PK2)), SyncMode.FULL_REFRESH, DestinationSyncMode.OVERWRITE); final AirbyteCatalog catalog1 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1), + getStreamAndConfig(STREAM2, streamConfiguration2))); final AirbyteCatalog catalog2 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1CursorDiff), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1CursorDiff), + getStreamAndConfig(STREAM2, streamConfiguration2))); final Set changedSd = connectionsHandler.getConfigurationDiff(catalog1, catalog2); assertFalse(changedSd.isEmpty()); assertEquals(1, changedSd.size()); - assertEquals(Set.of(new StreamDescriptor().name("stream1")), changedSd); + assertEquals(Set.of(new StreamDescriptor().name(STREAM1)), changedSd); } @Test void testDiffIfDifferentPrimaryKey() { final AirbyteStreamConfiguration streamConfiguration1 = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1")), + List.of(CURSOR1), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration1WithPkDiff = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1", "pk3")), + List.of(CURSOR1), + List.of(List.of(PK1, PK3)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration2 = getStreamConfiguration( - List.of("cursor2"), - List.of(List.of("pk2")), + List.of(CURSOR2), + List.of(List.of(PK2)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration2WithPkDiff = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1"), List.of("pk3")), + List.of(CURSOR1), + List.of(List.of(PK1), List.of(PK3)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteCatalog catalog1 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1), + getStreamAndConfig(STREAM2, streamConfiguration2))); final AirbyteCatalog catalog2 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1WithPkDiff), - getStreamAndConfig("stream2", streamConfiguration2WithPkDiff))); + getStreamAndConfig(STREAM1, streamConfiguration1WithPkDiff), + getStreamAndConfig(STREAM2, streamConfiguration2WithPkDiff))); final Set changedSd = connectionsHandler.getConfigurationDiff(catalog1, catalog2); assertFalse(changedSd.isEmpty()); assertEquals(2, changedSd.size()); Assertions.assertThat(changedSd) - .containsExactlyInAnyOrder(new StreamDescriptor().name("stream1"), new StreamDescriptor().name("stream2")); + .containsExactlyInAnyOrder(new StreamDescriptor().name(STREAM1), new StreamDescriptor().name(STREAM2)); } @Test void testDiffDifferentSyncMode() { final AirbyteStreamConfiguration streamConfiguration1 = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1")), + List.of(CURSOR1), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration1CursorDiff = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1")), + List.of(CURSOR1), + List.of(List.of(PK1)), SyncMode.FULL_REFRESH, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration2 = getStreamConfiguration( - List.of("cursor2"), - List.of(List.of("pk2")), + List.of(CURSOR2), + List.of(List.of(PK2)), SyncMode.FULL_REFRESH, DestinationSyncMode.OVERWRITE); final AirbyteCatalog catalog1 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1), + getStreamAndConfig(STREAM2, streamConfiguration2))); final AirbyteCatalog catalog2 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1CursorDiff), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1CursorDiff), + getStreamAndConfig(STREAM2, streamConfiguration2))); final Set changedSd = connectionsHandler.getConfigurationDiff(catalog1, catalog2); assertFalse(changedSd.isEmpty()); assertEquals(1, changedSd.size()); - assertEquals(Set.of(new StreamDescriptor().name("stream1")), changedSd); + assertEquals(Set.of(new StreamDescriptor().name(STREAM1)), changedSd); } @Test void testDiffDifferentDestinationSyncMode() { final AirbyteStreamConfiguration streamConfiguration1 = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1")), + List.of(CURSOR1), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND_DEDUP); final AirbyteStreamConfiguration streamConfiguration1CursorDiff = getStreamConfiguration( - List.of("cursor1"), - List.of(List.of("pk1")), + List.of(CURSOR1), + List.of(List.of(PK1)), SyncMode.INCREMENTAL, DestinationSyncMode.APPEND); final AirbyteStreamConfiguration streamConfiguration2 = getStreamConfiguration( - List.of("cursor2"), - List.of(List.of("pk2")), + List.of(CURSOR2), + List.of(List.of(PK2)), SyncMode.FULL_REFRESH, DestinationSyncMode.OVERWRITE); final AirbyteCatalog catalog1 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1), + getStreamAndConfig(STREAM2, streamConfiguration2))); final AirbyteCatalog catalog2 = new AirbyteCatalog() .streams( List.of( - getStreamAndConfig("stream1", streamConfiguration1CursorDiff), - getStreamAndConfig("stream2", streamConfiguration2))); + getStreamAndConfig(STREAM1, streamConfiguration1CursorDiff), + getStreamAndConfig(STREAM2, streamConfiguration2))); final Set changedSd = connectionsHandler.getConfigurationDiff(catalog1, catalog2); assertFalse(changedSd.isEmpty()); assertEquals(1, changedSd.size()); - assertEquals(Set.of(new StreamDescriptor().name("stream1")), changedSd); + assertEquals(Set.of(new StreamDescriptor().name(STREAM1)), changedSd); } private AirbyteStreamAndConfiguration getStreamAndConfig(final String name, final AirbyteStreamConfiguration config) { diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java index 516c5f06b48b..0ea12a1e0594 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java @@ -34,6 +34,10 @@ class OAuthHandlerTest { private OAuthHandler handler; private TrackingClient trackingClient; private HttpClient httpClient; + private static final String CLIENT_ID = "123"; + private static final String CLIENT_ID_KEY = "client_id"; + private static final String CLIENT_SECRET_KEY = "client_secret"; + private static final String CLIENT_SECRET = "hunter2"; @BeforeEach public void init() { @@ -47,8 +51,8 @@ public void init() { void setSourceInstancewideOauthParams() throws JsonValidationException, IOException { final UUID sourceDefId = UUID.randomUUID(); final Map params = new HashMap<>(); - params.put("client_id", "123"); - params.put("client_secret", "hunter2"); + params.put(CLIENT_ID_KEY, CLIENT_ID); + params.put(CLIENT_SECRET_KEY, CLIENT_SECRET); final SetInstancewideSourceOauthParamsRequestBody actualRequest = new SetInstancewideSourceOauthParamsRequestBody() .sourceDefinitionId(sourceDefId) @@ -66,8 +70,8 @@ void setSourceInstancewideOauthParams() throws JsonValidationException, IOExcept void resetSourceInstancewideOauthParams() throws JsonValidationException, IOException { final UUID sourceDefId = UUID.randomUUID(); final Map firstParams = new HashMap<>(); - firstParams.put("client_id", "123"); - firstParams.put("client_secret", "hunter2"); + firstParams.put(CLIENT_ID_KEY, CLIENT_ID); + firstParams.put(CLIENT_SECRET_KEY, CLIENT_SECRET); final SetInstancewideSourceOauthParamsRequestBody firstRequest = new SetInstancewideSourceOauthParamsRequestBody() .sourceDefinitionId(sourceDefId) .params(firstParams); @@ -78,8 +82,8 @@ void resetSourceInstancewideOauthParams() throws JsonValidationException, IOExce .thenReturn(Optional.of(new SourceOAuthParameter().withOauthParameterId(oauthParameterId))); final Map secondParams = new HashMap<>(); - secondParams.put("client_id", "456"); - secondParams.put("client_secret", "hunter3"); + secondParams.put(CLIENT_ID_KEY, "456"); + secondParams.put(CLIENT_SECRET_KEY, "hunter3"); final SetInstancewideSourceOauthParamsRequestBody secondRequest = new SetInstancewideSourceOauthParamsRequestBody() .sourceDefinitionId(sourceDefId) .params(secondParams); @@ -99,8 +103,8 @@ void resetSourceInstancewideOauthParams() throws JsonValidationException, IOExce void setDestinationInstancewideOauthParams() throws JsonValidationException, IOException { final UUID destinationDefId = UUID.randomUUID(); final Map params = new HashMap<>(); - params.put("client_id", "123"); - params.put("client_secret", "hunter2"); + params.put(CLIENT_ID_KEY, CLIENT_ID); + params.put(CLIENT_SECRET_KEY, CLIENT_SECRET); final SetInstancewideDestinationOauthParamsRequestBody actualRequest = new SetInstancewideDestinationOauthParamsRequestBody() .destinationDefinitionId(destinationDefId) @@ -118,8 +122,8 @@ void setDestinationInstancewideOauthParams() throws JsonValidationException, IOE void resetDestinationInstancewideOauthParams() throws JsonValidationException, IOException { final UUID destinationDefId = UUID.randomUUID(); final Map firstParams = new HashMap<>(); - firstParams.put("client_id", "123"); - firstParams.put("client_secret", "hunter2"); + firstParams.put(CLIENT_ID_KEY, CLIENT_ID); + firstParams.put(CLIENT_SECRET_KEY, CLIENT_SECRET); final SetInstancewideDestinationOauthParamsRequestBody firstRequest = new SetInstancewideDestinationOauthParamsRequestBody() .destinationDefinitionId(destinationDefId) .params(firstParams); @@ -130,8 +134,8 @@ void resetDestinationInstancewideOauthParams() throws JsonValidationException, I .thenReturn(Optional.of(new DestinationOAuthParameter().withOauthParameterId(oauthParameterId))); final Map secondParams = new HashMap<>(); - secondParams.put("client_id", "456"); - secondParams.put("client_secret", "hunter3"); + secondParams.put(CLIENT_ID_KEY, "456"); + secondParams.put(CLIENT_SECRET_KEY, "hunter3"); final SetInstancewideDestinationOauthParamsRequestBody secondRequest = new SetInstancewideDestinationOauthParamsRequestBody() .destinationDefinitionId(destinationDefId) .params(secondParams); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java index e41293a46425..2fab0ddfea4a 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java @@ -116,6 +116,13 @@ class WebBackendConnectionsHandlerTest { private EventRunner eventRunner; private ConfigRepository configRepository; + private static final String STREAM1 = "stream1"; + private static final String STREAM2 = "stream2"; + private static final String FIELD1 = "field1"; + private static final String FIELD2 = "field2"; + private static final String FIELD3 = "field3"; + private static final String FIELD5 = "field5"; + @BeforeEach void setup() throws IOException, JsonValidationException, ConfigNotFoundException { connectionsHandler = mock(ConnectionsHandler.class); @@ -922,27 +929,27 @@ void testUpdateSchemaWithDiscoveryFromEmpty() { final AirbyteCatalog original = new AirbyteCatalog().streams(List.of()); final AirbyteCatalog discovered = ConnectionHelpers.generateBasicApiCatalog(); discovered.getStreams().get(0).getStream() - .name("stream1") - .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of("field1", JsonSchemaType.STRING))) + .name(STREAM1) + .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD1, JsonSchemaType.STRING))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH)); discovered.getStreams().get(0).getConfig() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) - .aliasName("stream1"); + .aliasName(STREAM1); final AirbyteCatalog expected = ConnectionHelpers.generateBasicApiCatalog(); expected.getStreams().get(0).getStream() - .name("stream1") - .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of("field1", JsonSchemaType.STRING))) + .name(STREAM1) + .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD1, JsonSchemaType.STRING))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH)); expected.getStreams().get(0).getConfig() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) - .aliasName("stream1") + .aliasName(STREAM1) .setSelected(false); final AirbyteCatalog actual = WebBackendConnectionsHandler.updateSchemaWithDiscovery(original, discovered); @@ -955,44 +962,44 @@ void testUpdateSchemaWithDiscoveryResetStream() { final AirbyteCatalog original = ConnectionHelpers.generateBasicApiCatalog(); original.getStreams().get(0).getStream() .name("random-stream") - .defaultCursorField(List.of("field1")) + .defaultCursorField(List.of(FIELD1)) .jsonSchema(CatalogHelpers.fieldsToJsonSchema( - Field.of("field1", JsonSchemaType.NUMBER), - Field.of("field2", JsonSchemaType.NUMBER), - Field.of("field5", JsonSchemaType.STRING))) + Field.of(FIELD1, JsonSchemaType.NUMBER), + Field.of(FIELD2, JsonSchemaType.NUMBER), + Field.of(FIELD5, JsonSchemaType.STRING))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)); original.getStreams().get(0).getConfig() .syncMode(SyncMode.INCREMENTAL) - .cursorField(List.of("field1")) + .cursorField(List.of(FIELD1)) .destinationSyncMode(DestinationSyncMode.APPEND) .primaryKey(Collections.emptyList()) .aliasName("random_stream"); final AirbyteCatalog discovered = ConnectionHelpers.generateBasicApiCatalog(); discovered.getStreams().get(0).getStream() - .name("stream1") - .defaultCursorField(List.of("field3")) - .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of("field2", JsonSchemaType.STRING))) + .name(STREAM1) + .defaultCursorField(List.of(FIELD3)) + .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD2, JsonSchemaType.STRING))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)); discovered.getStreams().get(0).getConfig() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) - .aliasName("stream1"); + .aliasName(STREAM1); final AirbyteCatalog expected = ConnectionHelpers.generateBasicApiCatalog(); expected.getStreams().get(0).getStream() - .name("stream1") - .defaultCursorField(List.of("field3")) - .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of("field2", JsonSchemaType.STRING))) + .name(STREAM1) + .defaultCursorField(List.of(FIELD3)) + .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD2, JsonSchemaType.STRING))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)); expected.getStreams().get(0).getConfig() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) - .aliasName("stream1") + .aliasName(STREAM1) .setSelected(false); final AirbyteCatalog actual = WebBackendConnectionsHandler.updateSchemaWithDiscovery(original, discovered); @@ -1004,71 +1011,71 @@ void testUpdateSchemaWithDiscoveryResetStream() { void testUpdateSchemaWithDiscoveryMergeNewStream() { final AirbyteCatalog original = ConnectionHelpers.generateBasicApiCatalog(); original.getStreams().get(0).getStream() - .name("stream1") - .defaultCursorField(List.of("field1")) + .name(STREAM1) + .defaultCursorField(List.of(FIELD1)) .jsonSchema(CatalogHelpers.fieldsToJsonSchema( - Field.of("field1", JsonSchemaType.NUMBER), - Field.of("field2", JsonSchemaType.NUMBER), - Field.of("field5", JsonSchemaType.STRING))) + Field.of(FIELD1, JsonSchemaType.NUMBER), + Field.of(FIELD2, JsonSchemaType.NUMBER), + Field.of(FIELD5, JsonSchemaType.STRING))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)); original.getStreams().get(0).getConfig() .syncMode(SyncMode.INCREMENTAL) - .cursorField(List.of("field1")) + .cursorField(List.of(FIELD1)) .destinationSyncMode(DestinationSyncMode.APPEND) .primaryKey(Collections.emptyList()) .aliasName("renamed_stream"); final AirbyteCatalog discovered = ConnectionHelpers.generateBasicApiCatalog(); discovered.getStreams().get(0).getStream() - .name("stream1") - .defaultCursorField(List.of("field3")) - .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of("field2", JsonSchemaType.STRING))) + .name(STREAM1) + .defaultCursorField(List.of(FIELD3)) + .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD2, JsonSchemaType.STRING))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)); discovered.getStreams().get(0).getConfig() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) - .aliasName("stream1"); + .aliasName(STREAM1); final AirbyteStreamAndConfiguration newStream = ConnectionHelpers.generateBasicApiCatalog().getStreams().get(0); newStream.getStream() - .name("stream2") - .defaultCursorField(List.of("field5")) - .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of("field5", JsonSchemaType.BOOLEAN))) + .name(STREAM2) + .defaultCursorField(List.of(FIELD5)) + .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD5, JsonSchemaType.BOOLEAN))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH)); newStream.getConfig() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) - .aliasName("stream2"); + .aliasName(STREAM2); discovered.getStreams().add(newStream); final AirbyteCatalog expected = ConnectionHelpers.generateBasicApiCatalog(); expected.getStreams().get(0).getStream() - .name("stream1") - .defaultCursorField(List.of("field3")) - .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of("field2", JsonSchemaType.STRING))) + .name(STREAM1) + .defaultCursorField(List.of(FIELD3)) + .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD2, JsonSchemaType.STRING))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)); expected.getStreams().get(0).getConfig() .syncMode(SyncMode.INCREMENTAL) - .cursorField(List.of("field1")) + .cursorField(List.of(FIELD1)) .destinationSyncMode(DestinationSyncMode.APPEND) .primaryKey(Collections.emptyList()) .aliasName("renamed_stream") .setSelected(true); final AirbyteStreamAndConfiguration expectedNewStream = ConnectionHelpers.generateBasicApiCatalog().getStreams().get(0); expectedNewStream.getStream() - .name("stream2") - .defaultCursorField(List.of("field5")) - .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of("field5", JsonSchemaType.BOOLEAN))) + .name(STREAM2) + .defaultCursorField(List.of(FIELD5)) + .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD5, JsonSchemaType.BOOLEAN))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH)); expectedNewStream.getConfig() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) - .aliasName("stream2") + .aliasName(STREAM2) .setSelected(false); expected.getStreams().add(expectedNewStream); @@ -1097,27 +1104,27 @@ void testUpdateSchemaWithNamespacedStreams() { final AirbyteCatalog discovered = ConnectionHelpers.generateBasicApiCatalog(); discovered.getStreams().get(0).getStream() - .name("stream1") - .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of("field1", JsonSchemaType.STRING))) + .name(STREAM1) + .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD1, JsonSchemaType.STRING))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH)); discovered.getStreams().get(0).getConfig() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) - .aliasName("stream1"); + .aliasName(STREAM1); final AirbyteCatalog expected = ConnectionHelpers.generateBasicApiCatalog(); expected.getStreams().get(0).getStream() - .name("stream1") - .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of("field1", JsonSchemaType.STRING))) + .name(STREAM1) + .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(FIELD1, JsonSchemaType.STRING))) .supportedSyncModes(List.of(SyncMode.FULL_REFRESH)); expected.getStreams().get(0).getConfig() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) - .aliasName("stream1") + .aliasName(STREAM1) .setSelected(false); final AirbyteCatalog actual = WebBackendConnectionsHandler.updateSchemaWithDiscovery(original, discovered); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java index 9685c5a2b1f3..4a0f7b8dcf96 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WorkspacesHandlerTest.java @@ -60,6 +60,10 @@ class WorkspacesHandlerTest { private StandardWorkspace workspace; private WorkspacesHandler workspacesHandler; + private static final String TEST_EMAIL = "test@airbyte.io"; + private static final String TEST_WORKSPACE_NAME = "test workspace"; + private static final String TEST_WORKSPACE_SLUG = "test-workspace"; + @SuppressWarnings("unchecked") @BeforeEach void setUp() { @@ -76,9 +80,9 @@ private StandardWorkspace generateWorkspace() { return new StandardWorkspace() .withWorkspaceId(UUID.randomUUID()) .withCustomerId(UUID.randomUUID()) - .withEmail("test@airbyte.io") - .withName("test workspace") - .withSlug("test-workspace") + .withEmail(TEST_EMAIL) + .withName(TEST_WORKSPACE_NAME) + .withSlug(TEST_WORKSPACE_SLUG) .withInitialSetupComplete(false) .withDisplaySetupWizard(true) .withNews(false) @@ -113,7 +117,7 @@ void testCreateWorkspace() throws JsonValidationException, IOException { final WorkspaceCreate workspaceCreate = new WorkspaceCreate() .name("new workspace") - .email("test@airbyte.io") + .email(TEST_EMAIL) .news(false) .anonymousDataCollection(false) .securityUpdates(false) @@ -123,7 +127,7 @@ void testCreateWorkspace() throws JsonValidationException, IOException { final WorkspaceRead expectedRead = new WorkspaceRead() .workspaceId(uuid) .customerId(uuid) - .email("test@airbyte.io") + .email(TEST_EMAIL) .name("new workspace") .slug("new-workspace") .initialSetupComplete(false) @@ -150,7 +154,7 @@ void testCreateWorkspaceDuplicateSlug() throws JsonValidationException, IOExcept final WorkspaceCreate workspaceCreate = new WorkspaceCreate() .name(workspace.getName()) - .email("test@airbyte.io") + .email(TEST_EMAIL) .news(false) .anonymousDataCollection(false) .securityUpdates(false) @@ -160,7 +164,7 @@ void testCreateWorkspaceDuplicateSlug() throws JsonValidationException, IOExcept final WorkspaceRead expectedRead = new WorkspaceRead() .workspaceId(uuid) .customerId(uuid) - .email("test@airbyte.io") + .email(TEST_EMAIL) .name(workspace.getName()) .slug(workspace.getSlug()) .initialSetupComplete(false) @@ -257,9 +261,9 @@ void testGetWorkspace() throws JsonValidationException, ConfigNotFoundException, final WorkspaceRead workspaceRead = new WorkspaceRead() .workspaceId(workspace.getWorkspaceId()) .customerId(workspace.getCustomerId()) - .email("test@airbyte.io") - .name("test workspace") - .slug("test-workspace") + .email(TEST_EMAIL) + .name(TEST_WORKSPACE_NAME) + .slug(TEST_WORKSPACE_SLUG) .initialSetupComplete(false) .displaySetupWizard(true) .news(false) @@ -278,7 +282,7 @@ void testGetWorkspaceBySlug() throws JsonValidationException, ConfigNotFoundExce final WorkspaceRead workspaceRead = new WorkspaceRead() .workspaceId(workspace.getWorkspaceId()) .customerId(workspace.getCustomerId()) - .email("test@airbyte.io") + .email(TEST_EMAIL) .name(workspace.getName()) .slug(workspace.getSlug()) .initialSetupComplete(workspace.getInitialSetupComplete()) @@ -309,9 +313,9 @@ void testUpdateWorkspace() throws JsonValidationException, ConfigNotFoundExcepti final StandardWorkspace expectedWorkspace = new StandardWorkspace() .withWorkspaceId(workspace.getWorkspaceId()) .withCustomerId(workspace.getCustomerId()) - .withEmail("test@airbyte.io") - .withName("test workspace") - .withSlug("test-workspace") + .withEmail(TEST_EMAIL) + .withName(TEST_WORKSPACE_NAME) + .withSlug(TEST_WORKSPACE_SLUG) .withAnonymousDataCollection(true) .withSecurityUpdates(false) .withNews(false) @@ -331,9 +335,9 @@ void testUpdateWorkspace() throws JsonValidationException, ConfigNotFoundExcepti final WorkspaceRead expectedWorkspaceRead = new WorkspaceRead() .workspaceId(workspace.getWorkspaceId()) .customerId(workspace.getCustomerId()) - .email("test@airbyte.io") - .name("test workspace") - .slug("test-workspace") + .email(TEST_EMAIL) + .name(TEST_WORKSPACE_NAME) + .slug(TEST_WORKSPACE_SLUG) .initialSetupComplete(true) .displaySetupWizard(false) .news(false) @@ -356,7 +360,7 @@ void testUpdateWorkspaceNoNameUpdate() throws JsonValidationException, ConfigNot final StandardWorkspace expectedWorkspace = new StandardWorkspace() .withWorkspaceId(workspace.getWorkspaceId()) .withCustomerId(workspace.getCustomerId()) - .withEmail("test@airbyte.io") + .withEmail(TEST_EMAIL) .withName("New Workspace Name") .withSlug("new-workspace-name") .withAnonymousDataCollection(workspace.getAnonymousDataCollection()) @@ -376,7 +380,7 @@ void testUpdateWorkspaceNoNameUpdate() throws JsonValidationException, ConfigNot final WorkspaceRead expectedWorkspaceRead = new WorkspaceRead() .workspaceId(workspace.getWorkspaceId()) .customerId(workspace.getCustomerId()) - .email("test@airbyte.io") + .email(TEST_EMAIL) .name("New Workspace Name") .slug("new-workspace-name") .initialSetupComplete(workspace.getInitialSetupComplete()) diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java index e3c4ad38b78e..53deb0add230 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java @@ -45,16 +45,17 @@ public class ConnectionHelpers { private static final long BASIC_SCHEDULE_UNITS = 1L; private static final String BASIC_SCHEDULE_DATA_TIME_UNITS = "days"; private static final long BASIC_SCHEDULE_DATA_UNITS = 1L; + private static final String ONE_HUNDRED_G = "100g"; public static final StreamDescriptor STREAM_DESCRIPTOR = new StreamDescriptor().withName(STREAM_NAME); // only intended for unit tests, so intentionally set very high to ensure they aren't being used // elsewhere public static final io.airbyte.config.ResourceRequirements TESTING_RESOURCE_REQUIREMENTS = new io.airbyte.config.ResourceRequirements() - .withCpuLimit("100g") - .withCpuRequest("100g") - .withMemoryLimit("100g") - .withMemoryRequest("100g"); + .withCpuLimit(ONE_HUNDRED_G) + .withCpuRequest(ONE_HUNDRED_G) + .withMemoryLimit(ONE_HUNDRED_G) + .withMemoryRequest(ONE_HUNDRED_G); public static StandardSync generateSyncWithSourceId(final UUID sourceId) { final UUID connectionId = UUID.randomUUID(); diff --git a/airbyte-server/src/test/java/io/airbyte/server/services/AirbyteGithubStoreTest.java b/airbyte-server/src/test/java/io/airbyte/server/services/AirbyteGithubStoreTest.java index 7e049af4cd63..6f483ffdbdfe 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/services/AirbyteGithubStoreTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/services/AirbyteGithubStoreTest.java @@ -22,6 +22,10 @@ public class AirbyteGithubStoreTest { private static final Duration TIMEOUT = Duration.ofSeconds(1); + private static final String CONTENT_TYPE = "Content-Type"; + private static final String PLAIN_TEXT = "text/plain; charset=utf-8"; + private static final String CACHE_CONTROL = "Cache-Control"; + private static final String NO_CACHE = "no-cache"; private MockWebServer webServer; private AirbyteGithubStore githubStore; @@ -40,8 +44,8 @@ class FileUnusable { void testGetLatestSourcesWithNonJson() throws InterruptedException { final var nonjsonBody = "irrelevant text"; final var nonjsonResponse = new MockResponse().setResponseCode(200) - .addHeader("Content-Type", "text/plain; charset=utf-8") - .addHeader("Cache-Control", "no-cache") + .addHeader(CONTENT_TYPE, PLAIN_TEXT) + .addHeader(CACHE_CONTROL, NO_CACHE) .setBody(nonjsonBody); webServer.enqueue(nonjsonResponse); assertEquals(Collections.emptyList(), githubStore.getLatestSources()); @@ -51,8 +55,8 @@ void testGetLatestSourcesWithNonJson() throws InterruptedException { void testGetLatestSourcesWithWrongSchemaJson() throws InterruptedException { final var jsonBody = "{ json: 'validButWrongFormat' }"; final var jsonResponse = new MockResponse().setResponseCode(200) - .addHeader("Content-Type", "application/json; charset=utf-8") - .addHeader("Cache-Control", "no-cache") + .addHeader(CONTENT_TYPE, "application/json; charset=utf-8") + .addHeader(CACHE_CONTROL, NO_CACHE) .setBody(jsonBody); webServer.enqueue(jsonResponse); assertEquals(Collections.emptyList(), githubStore.getLatestSources()); @@ -62,8 +66,8 @@ void testGetLatestSourcesWithWrongSchemaJson() throws InterruptedException { void testGetLatestDestinationsWithNonJson() throws InterruptedException { final var nonjsonBody = "irrelevant text"; final var nonjsonResponse = new MockResponse().setResponseCode(200) - .addHeader("Content-Type", "text/plain; charset=utf-8") - .addHeader("Cache-Control", "no-cache") + .addHeader(CONTENT_TYPE, PLAIN_TEXT) + .addHeader(CACHE_CONTROL, NO_CACHE) .setBody(nonjsonBody); webServer.enqueue(nonjsonResponse); assertEquals(Collections.emptyList(), githubStore.getLatestDestinations()); @@ -73,8 +77,8 @@ void testGetLatestDestinationsWithNonJson() throws InterruptedException { void testGetLatestDestinationsWithWrongSchemaJson() throws InterruptedException { final var jsonBody = "{ json: 'validButWrongFormat' }"; final var jsonResponse = new MockResponse().setResponseCode(200) - .addHeader("Content-Type", "application/json; charset=utf-8") - .addHeader("Cache-Control", "no-cache") + .addHeader(CONTENT_TYPE, "application/json; charset=utf-8") + .addHeader(CACHE_CONTROL, NO_CACHE) .setBody(jsonBody); webServer.enqueue(jsonResponse); assertEquals(Collections.emptyList(), githubStore.getLatestDestinations()); @@ -130,8 +134,8 @@ class GetFile { void testReturn() throws IOException, InterruptedException { final var goodBody = "great day!"; final var goodResp = new MockResponse().setResponseCode(200) - .addHeader("Content-Type", "text/plain; charset=utf-8") - .addHeader("Cache-Control", "no-cache") + .addHeader(CONTENT_TYPE, PLAIN_TEXT) + .addHeader(CACHE_CONTROL, NO_CACHE) .setBody(goodBody); webServer.enqueue(goodResp); @@ -142,8 +146,8 @@ void testReturn() throws IOException, InterruptedException { @Test void testHttpTimeout() { final var timeoutResp = new MockResponse().setResponseCode(200) - .addHeader("Content-Type", "text/plain; charset=utf-8") - .addHeader("Cache-Control", "no-cache") + .addHeader(CONTENT_TYPE, PLAIN_TEXT) + .addHeader(CACHE_CONTROL, NO_CACHE) .setBody("") .setHeadersDelay(TIMEOUT.toSeconds() * 2, TimeUnit.SECONDS) .setBodyDelay(TIMEOUT.toSeconds() * 2, TimeUnit.SECONDS); diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/airbyte_test_container/AirbyteTestContainer.java b/airbyte-test-utils/src/main/java/io/airbyte/test/airbyte_test_container/AirbyteTestContainer.java index 4a88ed2276ab..826f414f50d1 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/airbyte_test_container/AirbyteTestContainer.java +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/airbyte_test_container/AirbyteTestContainer.java @@ -40,6 +40,7 @@ * { @link AirbyteTestContainer#stopRetainVolumes() }. It waits for Airbyte to be ready. It also * handles the nuances of configuring the Airbyte docker-compose configuration in test containers. */ +@SuppressWarnings("PMD.AvoidAccessibilityAlteration") public class AirbyteTestContainer { private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteTestContainer.class); diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java index 3b8f34f58abd..35c0db1de9c5 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java @@ -83,6 +83,8 @@ class AdvancedAcceptanceTests { private static final Logger LOGGER = LoggerFactory.getLogger(AdvancedAcceptanceTests.class); + private static final String TYPE = "type"; + private static final String COLUMN1 = "column1"; private static AirbyteAcceptanceTestHarness testHarness; private static AirbyteApiClient apiClient; @@ -157,7 +159,7 @@ void testCheckpointing() throws Exception { workspaceId, sourceDefinition.getSourceDefinitionId(), Jsons.jsonNode(ImmutableMap.builder() - .put("type", "EXCEPTION_AFTER_N") + .put(TYPE, "EXCEPTION_AFTER_N") .put("throw_after_n_records", 100) .build())); @@ -165,7 +167,7 @@ void testCheckpointing() throws Exception { "E2E Test Destination -" + UUID.randomUUID(), workspaceId, destinationDefinition.getDestinationDefinitionId(), - Jsons.jsonNode(ImmutableMap.of("type", "SILENT"))); + Jsons.jsonNode(ImmutableMap.of(TYPE, "SILENT"))); final String connectionName = "test-connection"; final UUID sourceId = source.getSourceId(); @@ -205,10 +207,10 @@ void testCheckpointing() throws Exception { // nature, we can't guarantee exactly what checkpoint will be registered. what we can do is send // enough messages to make sure that we checkpoint at least once. assertNotNull(connectionState.getState()); - assertTrue(connectionState.getState().get("column1").isInt()); - LOGGER.info("state value: {}", connectionState.getState().get("column1").asInt()); - assertTrue(connectionState.getState().get("column1").asInt() > 0); - assertEquals(0, connectionState.getState().get("column1").asInt() % 5); + assertTrue(connectionState.getState().get(COLUMN1).isInt()); + LOGGER.info("state value: {}", connectionState.getState().get(COLUMN1).asInt()); + assertTrue(connectionState.getState().get(COLUMN1).asInt() > 0); + assertEquals(0, connectionState.getState().get(COLUMN1).asInt() % 5); } @RetryingTest(3) @@ -246,7 +248,7 @@ void testBackpressure() throws Exception { workspaceId, sourceDefinition.getSourceDefinitionId(), Jsons.jsonNode(ImmutableMap.builder() - .put("type", "INFINITE_FEED") + .put(TYPE, "INFINITE_FEED") .put("max_records", 5000) .build())); @@ -255,7 +257,7 @@ void testBackpressure() throws Exception { workspaceId, destinationDefinition.getDestinationDefinitionId(), Jsons.jsonNode(ImmutableMap.builder() - .put("type", "THROTTLED") + .put(TYPE, "THROTTLED") .put("millis_per_record", 1) .build())); diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java index f07de6b512a0..5c4089098d7f 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java @@ -134,6 +134,22 @@ class BasicAcceptanceTests { private static UUID workspaceId; private static PostgreSQLContainer sourcePsql; + private static final String TYPE = "type"; + private static final String PUBLIC = "public"; + private static final String E2E_TEST_SOURCE = "E2E Test Source -"; + private static final String INFINITE_FEED = "INFINITE_FEED"; + private static final String MESSAGE_INTERVAL = "message_interval"; + private static final String MAX_RECORDS = "max_records"; + private static final String TEST_CONNECTION = "test-connection"; + private static final String STATE_AFTER_SYNC_ONE = "state after sync 1: {}"; + private static final String STATE_AFTER_SYNC_TWO = "state after sync 2: {}"; + private static final String GERALT = "geralt"; + private static final String NAME = "name"; + private static final String VALUE = "value"; + private static final String LOCATION = "location"; + private static final String FIELD = "field"; + private static final String ID_AND_NAME = "id_and_name"; + @BeforeAll static void init() throws URISyntaxException, IOException, InterruptedException, ApiException { apiClient = new AirbyteApiClient( @@ -283,15 +299,15 @@ void testDiscoverSourceSchema() throws ApiException { final AirbyteCatalog actual = testHarness.discoverSourceSchema(sourceId); final Map> fields = ImmutableMap.of( - COLUMN_ID, ImmutableMap.of("type", DataType.NUMBER.getValue(), "airbyte_type", "integer"), - COLUMN_NAME, ImmutableMap.of("type", DataType.STRING.getValue())); + COLUMN_ID, ImmutableMap.of(TYPE, DataType.NUMBER.getValue(), "airbyte_type", "integer"), + COLUMN_NAME, ImmutableMap.of(TYPE, DataType.STRING.getValue())); final JsonNode jsonSchema = Jsons.jsonNode(ImmutableMap.builder() - .put("type", "object") + .put(TYPE, "object") .put("properties", fields) .build()); final AirbyteStream stream = new AirbyteStream() .name(STREAM_NAME) - .namespace("public") + .namespace(PUBLIC) .jsonSchema(jsonSchema) .sourceDefinedCursor(null) .defaultCursorField(Collections.emptyList()) @@ -342,16 +358,15 @@ void testCancelSync() throws Exception { final SourceDefinitionRead sourceDefinition = testHarness.createE2eSourceDefinition(); final SourceRead source = testHarness.createSource( - "E2E Test Source -" + UUID.randomUUID(), + E2E_TEST_SOURCE + UUID.randomUUID(), workspaceId, sourceDefinition.getSourceDefinitionId(), Jsons.jsonNode(ImmutableMap.builder() - .put("type", "INFINITE_FEED") - .put("message_interval", 1000) - .put("max_records", Duration.ofMinutes(5).toSeconds()) + .put(TYPE, INFINITE_FEED) + .put(MESSAGE_INTERVAL, 1000) + .put(MAX_RECORDS, Duration.ofMinutes(5).toSeconds()) .build())); - final String connectionName = "test-connection"; final UUID sourceId = source.getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -360,7 +375,7 @@ void testCancelSync() throws Exception { final DestinationSyncMode destinationSyncMode = DestinationSyncMode.OVERWRITE; catalog.getStreams().forEach(s -> s.getConfig().syncMode(syncMode).destinationSyncMode(destinationSyncMode)); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); final JobInfoRead connectionSyncRead = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); // wait to get out of PENDING @@ -374,7 +389,6 @@ void testCancelSync() throws Exception { @Test @Order(8) void testScheduledSync() throws Exception { - final String connectionName = "test-connection"; final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -386,7 +400,7 @@ void testScheduledSync() throws Exception { catalog.getStreams().forEach(s -> s.getConfig().syncMode(syncMode).destinationSyncMode(destinationSyncMode)); final var conn = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, connectionSchedule); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, connectionSchedule); // When a new connection is created, Airbyte might sync it immediately (before the sync interval). // Then it will wait the sync interval. @@ -410,7 +424,6 @@ void testMultipleSchemasAndTablesSync() throws Exception { // create tables in another schema PostgreSQLContainerHelper.runSqlScript(MountableFile.forClasspathResource("postgres_second_schema_multiple_tables.sql"), sourcePsql); - final String connectionName = "test-connection"; final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -420,7 +433,7 @@ void testMultipleSchemasAndTablesSync() throws Exception { final DestinationSyncMode destinationSyncMode = DestinationSyncMode.OVERWRITE; catalog.getStreams().forEach(s -> s.getConfig().syncMode(syncMode).destinationSyncMode(destinationSyncMode)); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); final JobInfoRead connectionSyncRead = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead.getJob()); testHarness.assertSourceAndDestinationDbInSync(false); @@ -432,7 +445,6 @@ void testMultipleSchemasSameTablesSync() throws Exception { // create tables in another schema PostgreSQLContainerHelper.runSqlScript(MountableFile.forClasspathResource("postgres_separate_schema_same_table.sql"), sourcePsql); - final String connectionName = "test-connection"; final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -442,7 +454,7 @@ void testMultipleSchemasSameTablesSync() throws Exception { final DestinationSyncMode destinationSyncMode = DestinationSyncMode.OVERWRITE; catalog.getStreams().forEach(s -> s.getConfig().syncMode(syncMode).destinationSyncMode(destinationSyncMode)); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); final JobInfoRead connectionSyncRead = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead.getJob()); @@ -452,7 +464,6 @@ void testMultipleSchemasSameTablesSync() throws Exception { @Test @Order(11) void testIncrementalDedupeSync() throws Exception { - final String connectionName = "test-connection"; final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -465,7 +476,7 @@ void testIncrementalDedupeSync() throws Exception { .destinationSyncMode(destinationSyncMode) .primaryKey(List.of(List.of(COLUMN_NAME)))); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); // sync from start final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() @@ -489,7 +500,7 @@ void testIncrementalDedupeSync() throws Exception { .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - testHarness.assertRawDestinationContains(expectedRawRecords, new SchemaTableNamePair("public", STREAM_NAME)); + testHarness.assertRawDestinationContains(expectedRawRecords, new SchemaTableNamePair(PUBLIC, STREAM_NAME)); testHarness.assertNormalizedDestinationContains(expectedNormalizedRecords); } @@ -497,7 +508,6 @@ void testIncrementalDedupeSync() throws Exception { @Order(12) void testIncrementalSync() throws Exception { LOGGER.info("Starting testIncrementalSync()"); - final String connectionName = "test-connection"; final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -517,13 +527,13 @@ void testIncrementalSync() throws Exception { .cursorField(List.of(COLUMN_ID)) .destinationSyncMode(destinationSyncMode)); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); LOGGER.info("Beginning testIncrementalSync() sync 1"); final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead1.getJob()); - LOGGER.info("state after sync 1: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -531,7 +541,7 @@ void testIncrementalSync() throws Exception { final Database source = testHarness.getSourceDatabase(); // get contents of source before mutating records. final List expectedRecords = testHarness.retrieveSourceRecords(source, STREAM_NAME); - expectedRecords.add(Jsons.jsonNode(ImmutableMap.builder().put(COLUMN_ID, 6).put(COLUMN_NAME, "geralt").build())); + expectedRecords.add(Jsons.jsonNode(ImmutableMap.builder().put(COLUMN_ID, 6).put(COLUMN_NAME, GERALT).build())); // add a new record source.query(ctx -> ctx.execute("INSERT INTO id_and_name(id, name) VALUES(6, 'geralt')")); // mutate a record that was already synced with out updating its cursor value. if we are actually @@ -543,9 +553,9 @@ void testIncrementalSync() throws Exception { final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - LOGGER.info("state after sync 2: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); - testHarness.assertRawDestinationContains(expectedRecords, new SchemaTableNamePair("public", STREAM_NAME)); + testHarness.assertRawDestinationContains(expectedRecords, new SchemaTableNamePair(PUBLIC, STREAM_NAME)); // reset back to no data. @@ -556,7 +566,7 @@ void testIncrementalSync() throws Exception { LOGGER.info("state after reset: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); - testHarness.assertRawDestinationContains(Collections.emptyList(), new SchemaTableNamePair("public", + testHarness.assertRawDestinationContains(Collections.emptyList(), new SchemaTableNamePair(PUBLIC, STREAM_NAME)); // sync one more time. verify it is the equivalent of a full refresh. @@ -573,7 +583,6 @@ void testIncrementalSync() throws Exception { @Test @Order(13) void testDeleteConnection() throws Exception { - final String connectionName = "test-connection"; final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -587,7 +596,7 @@ void testDeleteConnection() throws Exception { .primaryKey(List.of(List.of(COLUMN_NAME)))); UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); final JobInfoRead connectionSyncRead = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitWhileJobHasStatus(apiClient.getJobsApi(), connectionSyncRead.getJob(), Set.of(JobStatus.RUNNING)); @@ -614,7 +623,7 @@ void testDeleteConnection() throws Exception { // test deletion of connection when temporal workflow is in a bad state LOGGER.info("Testing connection deletion when temporal is in a terminal state"); connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); testHarness.terminateTemporalWorkflow(connectionId); @@ -638,7 +647,6 @@ void testUpdateConnectionWhenWorkflowUnreachable() throws Exception { // schedule is out of scope for the issue (https://github.com/airbytehq/airbyte/issues/11215). This // test just ensures that the underlying workflow // is running after the update method is called. - final String connectionName = "test-connection"; final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -651,7 +659,7 @@ void testUpdateConnectionWhenWorkflowUnreachable() throws Exception { LOGGER.info("Testing connection update when temporal is in a terminal state"); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); testHarness.terminateTemporalWorkflow(connectionId); @@ -671,16 +679,15 @@ void testUpdateConnectionWhenWorkflowUnreachable() throws Exception { void testManualSyncRepairsWorkflowWhenWorkflowUnreachable() throws Exception { // This test only covers the specific behavior of updating a connection that does not have an // underlying temporal workflow. - final String connectionName = "test-connection"; final SourceDefinitionRead sourceDefinition = testHarness.createE2eSourceDefinition(); final SourceRead source = testHarness.createSource( - "E2E Test Source -" + UUID.randomUUID(), + E2E_TEST_SOURCE + UUID.randomUUID(), workspaceId, sourceDefinition.getSourceDefinitionId(), Jsons.jsonNode(ImmutableMap.builder() - .put("type", "INFINITE_FEED") - .put("max_records", 5000) - .put("message_interval", 100) + .put(TYPE, INFINITE_FEED) + .put(MAX_RECORDS, 5000) + .put(MESSAGE_INTERVAL, 100) .build())); final UUID sourceId = source.getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); @@ -694,7 +701,7 @@ void testManualSyncRepairsWorkflowWhenWorkflowUnreachable() throws Exception { LOGGER.info("Testing manual sync when temporal is in a terminal state"); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); LOGGER.info("Starting first manual sync"); final JobInfoRead firstJobInfo = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); @@ -721,7 +728,6 @@ void testManualSyncRepairsWorkflowWhenWorkflowUnreachable() throws Exception { void testResetConnectionRepairsWorkflowWhenWorkflowUnreachable() throws Exception { // This test only covers the specific behavior of updating a connection that does not have an // underlying temporal workflow. - final String connectionName = "test-connection"; final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -734,7 +740,7 @@ void testResetConnectionRepairsWorkflowWhenWorkflowUnreachable() throws Exceptio LOGGER.info("Testing reset connection when temporal is in a terminal state"); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); testHarness.terminateTemporalWorkflow(connectionId); @@ -748,16 +754,15 @@ void testResetCancelsRunningSync() throws Exception { final SourceDefinitionRead sourceDefinition = testHarness.createE2eSourceDefinition(); final SourceRead source = testHarness.createSource( - "E2E Test Source -" + UUID.randomUUID(), + E2E_TEST_SOURCE + UUID.randomUUID(), workspaceId, sourceDefinition.getSourceDefinitionId(), Jsons.jsonNode(ImmutableMap.builder() - .put("type", "INFINITE_FEED") - .put("message_interval", 1000) - .put("max_records", Duration.ofMinutes(5).toSeconds()) + .put(TYPE, INFINITE_FEED) + .put(MESSAGE_INTERVAL, 1000) + .put(MAX_RECORDS, Duration.ofMinutes(5).toSeconds()) .build())); - final String connectionName = "test-connection"; final UUID sourceId = source.getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -766,7 +771,7 @@ void testResetCancelsRunningSync() throws Exception { final DestinationSyncMode destinationSyncMode = DestinationSyncMode.OVERWRITE; catalog.getStreams().forEach(s -> s.getConfig().syncMode(syncMode).destinationSyncMode(destinationSyncMode)); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); final JobInfoRead connectionSyncRead = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); // wait to get out of PENDING @@ -786,7 +791,6 @@ void testResetCancelsRunningSync() throws Exception { @Test void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Exception { LOGGER.info("Starting {}", testInfo.getDisplayName()); - final String connectionName = "test-connection"; final SourceRead source = testHarness.createPostgresSource(); final UUID sourceId = source.getSourceId(); final UUID sourceDefinitionId = source.getSourceDefinitionId(); @@ -809,13 +813,13 @@ void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Except .cursorField(List.of(COLUMN_ID)) .destinationSyncMode(DestinationSyncMode.APPEND)); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); LOGGER.info("Beginning {} sync 1", testInfo.getDisplayName()); final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead1.getJob()); - LOGGER.info("state after sync 1: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -827,7 +831,7 @@ void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Except final Database sourceDatabase = testHarness.getSourceDatabase(); // get contents of source before mutating records. final List expectedRecords = testHarness.retrieveSourceRecords(sourceDatabase, STREAM_NAME); - expectedRecords.add(Jsons.jsonNode(ImmutableMap.builder().put(COLUMN_ID, 6).put(COLUMN_NAME, "geralt").build())); + expectedRecords.add(Jsons.jsonNode(ImmutableMap.builder().put(COLUMN_ID, 6).put(COLUMN_NAME, GERALT).build())); // add a new record sourceDatabase.query(ctx -> ctx.execute("INSERT INTO id_and_name(id, name) VALUES(6, 'geralt')")); // mutate a record that was already synced with out updating its cursor value. if we are actually @@ -839,9 +843,9 @@ void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Except final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - LOGGER.info("state after sync 2: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); - testHarness.assertRawDestinationContains(expectedRecords, new SchemaTableNamePair("public", STREAM_NAME)); + testHarness.assertRawDestinationContains(expectedRecords, new SchemaTableNamePair(PUBLIC, STREAM_NAME)); // reset back to no data. LOGGER.info("Starting {} reset", testInfo.getDisplayName()); @@ -851,7 +855,7 @@ void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Except LOGGER.info("state after reset: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); - testHarness.assertRawDestinationContains(Collections.emptyList(), new SchemaTableNamePair("public", + testHarness.assertRawDestinationContains(Collections.emptyList(), new SchemaTableNamePair(PUBLIC, STREAM_NAME)); // sync one more time. verify it is the equivalent of a full refresh. @@ -868,14 +872,13 @@ void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Except assertNotNull(state.getStreamState()); assertEquals(1, state.getStreamState().size()); final StreamState idAndNameState = state.getStreamState().get(0); - assertEquals(new StreamDescriptor().namespace("public").name(STREAM_NAME), idAndNameState.getStreamDescriptor()); + assertEquals(new StreamDescriptor().namespace(PUBLIC).name(STREAM_NAME), idAndNameState.getStreamDescriptor()); assertEquals(Jsons.deserialize(expectedState), idAndNameState.getStreamState()); } @Test void testSyncAfterUpgradeToPerStreamStateWithNoNewData(final TestInfo testInfo) throws Exception { LOGGER.info("Starting {}", testInfo.getDisplayName()); - final String connectionName = "test-connection"; final SourceRead source = testHarness.createPostgresSource(); final UUID sourceId = source.getSourceId(); final UUID sourceDefinitionId = source.getSourceDefinitionId(); @@ -898,13 +901,13 @@ void testSyncAfterUpgradeToPerStreamStateWithNoNewData(final TestInfo testInfo) .cursorField(List.of(COLUMN_ID)) .destinationSyncMode(DestinationSyncMode.APPEND)); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); LOGGER.info("Beginning {} sync 1", testInfo.getDisplayName()); final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead1.getJob()); - LOGGER.info("state after sync 1: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -917,7 +920,7 @@ void testSyncAfterUpgradeToPerStreamStateWithNoNewData(final TestInfo testInfo) final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - LOGGER.info("state after sync 2: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); final JobInfoRead syncJob = apiClient.getJobsApi().getJobInfo(new JobIdRequestBody().id(connectionSyncRead2.getJob().getId())); final Optional result = syncJob.getAttempts().stream() @@ -939,15 +942,15 @@ void testResetAllWhenSchemaIsModified() throws Exception { final Database sourceDb = testHarness.getSourceDatabase(); final Database destDb = testHarness.getDestinationDatabase(); sourceDb.query(ctx -> { - ctx.createTableIfNotExists(sourceTable1).columns(DSL.field("name", SQLDataType.VARCHAR)).execute(); + ctx.createTableIfNotExists(sourceTable1).columns(DSL.field(NAME, SQLDataType.VARCHAR)).execute(); ctx.truncate(sourceTable1).execute(); - ctx.insertInto(DSL.table(sourceTable1)).columns(DSL.field("name")).values("john").execute(); - ctx.insertInto(DSL.table(sourceTable1)).columns(DSL.field("name")).values("bob").execute(); + ctx.insertInto(DSL.table(sourceTable1)).columns(DSL.field(NAME)).values("john").execute(); + ctx.insertInto(DSL.table(sourceTable1)).columns(DSL.field(NAME)).values("bob").execute(); - ctx.createTableIfNotExists(sourceTable2).columns(DSL.field("value", SQLDataType.VARCHAR)).execute(); + ctx.createTableIfNotExists(sourceTable2).columns(DSL.field(VALUE, SQLDataType.VARCHAR)).execute(); ctx.truncate(sourceTable2).execute(); - ctx.insertInto(DSL.table(sourceTable2)).columns(DSL.field("value")).values("v1").execute(); - ctx.insertInto(DSL.table(sourceTable2)).columns(DSL.field("value")).values("v2").execute(); + ctx.insertInto(DSL.table(sourceTable2)).columns(DSL.field(VALUE)).values("v1").execute(); + ctx.insertInto(DSL.table(sourceTable2)).columns(DSL.field(VALUE)).values("v2").execute(); return null; }); @@ -989,19 +992,19 @@ void testResetAllWhenSchemaIsModified() throws Exception { LOGGER.info("Modifying source tables"); sourceDb.query(ctx -> { // Adding a new rows to make sure we sync more data. - ctx.insertInto(DSL.table(sourceTable1)).columns(DSL.field("name")).values("alice").execute(); - ctx.insertInto(DSL.table(sourceTable2)).columns(DSL.field("value")).values("v3").execute(); + ctx.insertInto(DSL.table(sourceTable1)).columns(DSL.field(NAME)).values("alice").execute(); + ctx.insertInto(DSL.table(sourceTable2)).columns(DSL.field(VALUE)).values("v3").execute(); // The removed rows should no longer be in the destination since we expect a full reset - ctx.deleteFrom(DSL.table(sourceTable1)).where(DSL.field("name").eq("john")).execute(); - ctx.deleteFrom(DSL.table(sourceTable2)).where(DSL.field("value").eq("v2")).execute(); + ctx.deleteFrom(DSL.table(sourceTable1)).where(DSL.field(NAME).eq("john")).execute(); + ctx.deleteFrom(DSL.table(sourceTable2)).where(DSL.field(VALUE).eq("v2")).execute(); // Adding a new table to trigger reset from the update connection API - ctx.createTableIfNotExists(sourceTable3).columns(DSL.field("location", SQLDataType.VARCHAR)).execute(); + ctx.createTableIfNotExists(sourceTable3).columns(DSL.field(LOCATION, SQLDataType.VARCHAR)).execute(); ctx.truncate(sourceTable3).execute(); - ctx.insertInto(DSL.table(sourceTable3)).columns(DSL.field("location")).values("home").execute(); - ctx.insertInto(DSL.table(sourceTable3)).columns(DSL.field("location")).values("work").execute(); - ctx.insertInto(DSL.table(sourceTable3)).columns(DSL.field("location")).values("space").execute(); + ctx.insertInto(DSL.table(sourceTable3)).columns(DSL.field(LOCATION)).values("home").execute(); + ctx.insertInto(DSL.table(sourceTable3)).columns(DSL.field(LOCATION)).values("work").execute(); + ctx.insertInto(DSL.table(sourceTable3)).columns(DSL.field(LOCATION)).values("space").execute(); return null; }); @@ -1077,7 +1080,6 @@ void testIncrementalSyncMultipleStreams() throws Exception { PostgreSQLContainerHelper.runSqlScript(MountableFile.forClasspathResource("postgres_second_schema_multiple_tables.sql"), sourcePsql); - final String connectionName = "test-connection"; final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -1099,13 +1101,13 @@ void testIncrementalSyncMultipleStreams() throws Exception { .cursorField(List.of(COLUMN_ID)) .destinationSyncMode(destinationSyncMode)); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); LOGGER.info("Beginning testIncrementalSync() sync 1"); final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead1.getJob()); - LOGGER.info("state after sync 1: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_ONE, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); @@ -1117,9 +1119,9 @@ void testIncrementalSyncMultipleStreams() throws Exception { testHarness.retrieveSourceRecords(source, STAGING_SCHEMA_NAME + "." + COOL_EMPLOYEES_TABLE_NAME); final List expectedRecordsAwesomePeople = testHarness.retrieveSourceRecords(source, STAGING_SCHEMA_NAME + "." + AWESOME_PEOPLE_TABLE_NAME); - expectedRecordsIdAndName.add(Jsons.jsonNode(ImmutableMap.builder().put(COLUMN_ID, 6).put(COLUMN_NAME, "geralt").build())); - expectedRecordsCoolEmployees.add(Jsons.jsonNode(ImmutableMap.builder().put(COLUMN_ID, 6).put(COLUMN_NAME, "geralt").build())); - expectedRecordsAwesomePeople.add(Jsons.jsonNode(ImmutableMap.builder().put(COLUMN_ID, 3).put(COLUMN_NAME, "geralt").build())); + expectedRecordsIdAndName.add(Jsons.jsonNode(ImmutableMap.builder().put(COLUMN_ID, 6).put(COLUMN_NAME, GERALT).build())); + expectedRecordsCoolEmployees.add(Jsons.jsonNode(ImmutableMap.builder().put(COLUMN_ID, 6).put(COLUMN_NAME, GERALT).build())); + expectedRecordsAwesomePeople.add(Jsons.jsonNode(ImmutableMap.builder().put(COLUMN_ID, 3).put(COLUMN_NAME, GERALT).build())); // add a new record to each table source.query(ctx -> ctx.execute("INSERT INTO id_and_name(id, name) VALUES(6, 'geralt')")); source.query(ctx -> ctx.execute("INSERT INTO staging.cool_employees(id, name) VALUES(6, 'geralt')")); @@ -1135,7 +1137,7 @@ void testIncrementalSyncMultipleStreams() throws Exception { final JobInfoRead connectionSyncRead2 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead2.getJob()); - LOGGER.info("state after sync 2: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); + LOGGER.info(STATE_AFTER_SYNC_TWO, apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); testHarness.assertRawDestinationContains(expectedRecordsIdAndName, new SchemaTableNamePair(PUBLIC_SCHEMA_NAME, STREAM_NAME)); testHarness.assertRawDestinationContains(expectedRecordsCoolEmployees, new SchemaTableNamePair(STAGING_SCHEMA_NAME, COOL_EMPLOYEES_TABLE_NAME)); @@ -1150,7 +1152,7 @@ void testIncrementalSyncMultipleStreams() throws Exception { LOGGER.info("state after reset: {}", apiClient.getConnectionApi().getState(new ConnectionIdRequestBody().connectionId(connectionId))); - testHarness.assertRawDestinationContains(Collections.emptyList(), new SchemaTableNamePair("public", + testHarness.assertRawDestinationContains(Collections.emptyList(), new SchemaTableNamePair(PUBLIC, STREAM_NAME)); // sync one more time. verify it is the equivalent of a full refresh. @@ -1169,7 +1171,6 @@ void testMultipleSchemasAndTablesSyncAndReset() throws Exception { // create tables in another schema PostgreSQLContainerHelper.runSqlScript(MountableFile.forClasspathResource("postgres_second_schema_multiple_tables.sql"), sourcePsql); - final String connectionName = "test-connection"; final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); final UUID operationId = testHarness.createOperation().getOperationId(); @@ -1179,7 +1180,7 @@ void testMultipleSchemasAndTablesSyncAndReset() throws Exception { final DestinationSyncMode destinationSyncMode = DestinationSyncMode.OVERWRITE; catalog.getStreams().forEach(s -> s.getConfig().syncMode(syncMode).destinationSyncMode(destinationSyncMode)); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, List.of(operationId), catalog, null).getConnectionId(); final JobInfoRead connectionSyncRead = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead.getJob()); testHarness.assertSourceAndDestinationDbInSync(false); @@ -1197,10 +1198,10 @@ void testPartialResetResetAllWhenSchemaIsModified(final TestInfo testInfo) throw final Database sourceDb = testHarness.getSourceDatabase(); sourceDb.query(ctx -> { ctx.createTableIfNotExists(additionalTable) - .columns(DSL.field("id", SQLDataType.INTEGER), DSL.field("field", SQLDataType.VARCHAR)).execute(); + .columns(DSL.field("id", SQLDataType.INTEGER), DSL.field(FIELD, SQLDataType.VARCHAR)).execute(); ctx.truncate(additionalTable).execute(); - ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field("field")).values(1, "1").execute(); - ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field("field")).values(2, "2").execute(); + ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field(FIELD)).values(1, "1").execute(); + ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field(FIELD)).values(2, "2").execute(); return null; }); UUID sourceId = testHarness.createPostgresSource().getSourceId(); @@ -1222,8 +1223,8 @@ void testPartialResetResetAllWhenSchemaIsModified(final TestInfo testInfo) throw testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); assertStreamStateContainsStream(connection.getConnectionId(), List.of( - new StreamDescriptor().name("id_and_name").namespace("public"), - new StreamDescriptor().name(additionalTable).namespace("public"))); + new StreamDescriptor().name(ID_AND_NAME).namespace(PUBLIC), + new StreamDescriptor().name(additionalTable).namespace(PUBLIC))); LOGGER.info("Initial sync ran, now running an update with a stream being removed."); @@ -1244,7 +1245,7 @@ void testPartialResetResetAllWhenSchemaIsModified(final TestInfo testInfo) throw // We do not check that the source and the dest are in sync here because removing a stream doesn't // remove that assertStreamStateContainsStream(connection.getConnectionId(), List.of( - new StreamDescriptor().name("id_and_name").namespace("public"))); + new StreamDescriptor().name(ID_AND_NAME).namespace(PUBLIC))); LOGGER.info("Remove done, now running an update with a stream being added."); @@ -1254,10 +1255,10 @@ void testPartialResetResetAllWhenSchemaIsModified(final TestInfo testInfo) throw */ sourceDb.query(ctx -> { ctx.createTableIfNotExists(additionalTable) - .columns(DSL.field("id", SQLDataType.INTEGER), DSL.field("field", SQLDataType.VARCHAR)).execute(); + .columns(DSL.field("id", SQLDataType.INTEGER), DSL.field(FIELD, SQLDataType.VARCHAR)).execute(); ctx.truncate(additionalTable).execute(); - ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field("field")).values(3, "3").execute(); - ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field("field")).values(4, "4").execute(); + ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field(FIELD)).values(3, "3").execute(); + ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field(FIELD)).values(4, "4").execute(); return null; }); @@ -1273,8 +1274,8 @@ void testPartialResetResetAllWhenSchemaIsModified(final TestInfo testInfo) throw // remove that testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); assertStreamStateContainsStream(connection.getConnectionId(), List.of( - new StreamDescriptor().name("id_and_name").namespace("public"), - new StreamDescriptor().name(additionalTable).namespace("public"))); + new StreamDescriptor().name(ID_AND_NAME).namespace(PUBLIC), + new StreamDescriptor().name(additionalTable).namespace(PUBLIC))); LOGGER.info("Addition done, now running an update with a stream being updated."); @@ -1282,12 +1283,12 @@ void testPartialResetResetAllWhenSchemaIsModified(final TestInfo testInfo) throw sourceDb.query(ctx -> { ctx.dropTableIfExists(additionalTable).execute(); ctx.createTableIfNotExists(additionalTable) - .columns(DSL.field("id", SQLDataType.INTEGER), DSL.field("field", SQLDataType.VARCHAR), DSL.field("another_field", SQLDataType.VARCHAR)) + .columns(DSL.field("id", SQLDataType.INTEGER), DSL.field(FIELD, SQLDataType.VARCHAR), DSL.field("another_field", SQLDataType.VARCHAR)) .execute(); ctx.truncate(additionalTable).execute(); - ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field("field"), DSL.field("another_field")).values(3, "3", "three") + ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field(FIELD), DSL.field("another_field")).values(3, "3", "three") .execute(); - ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field("field"), DSL.field("another_field")).values(4, "4", "four") + ctx.insertInto(DSL.table(additionalTable)).columns(DSL.field("id"), DSL.field(FIELD), DSL.field("another_field")).values(4, "4", "four") .execute(); return null; }); @@ -1304,8 +1305,8 @@ void testPartialResetResetAllWhenSchemaIsModified(final TestInfo testInfo) throw // remove that testHarness.assertSourceAndDestinationDbInSync(WITHOUT_SCD_TABLE); assertStreamStateContainsStream(connection.getConnectionId(), List.of( - new StreamDescriptor().name("id_and_name").namespace("public"), - new StreamDescriptor().name(additionalTable).namespace("public"))); + new StreamDescriptor().name(ID_AND_NAME).namespace(PUBLIC), + new StreamDescriptor().name(additionalTable).namespace(PUBLIC))); } @@ -1347,13 +1348,13 @@ void testFailureTimeout() throws Exception { final DestinationDefinitionRead destinationDefinition = testHarness.createE2eDestinationDefinition(); final SourceRead source = testHarness.createSource( - "E2E Test Source -" + UUID.randomUUID(), + E2E_TEST_SOURCE + UUID.randomUUID(), workspaceId, sourceDefinition.getSourceDefinitionId(), Jsons.jsonNode(ImmutableMap.builder() - .put("type", "INFINITE_FEED") - .put("max_records", 1000) - .put("message_interval", 100) + .put(TYPE, INFINITE_FEED) + .put(MAX_RECORDS, 1000) + .put(MESSAGE_INTERVAL, 100) .build())); // Destination fails after processing 5 messages, so the job should fail after the graceful close @@ -1363,17 +1364,16 @@ void testFailureTimeout() throws Exception { workspaceId, destinationDefinition.getDestinationDefinitionId(), Jsons.jsonNode(ImmutableMap.builder() - .put("type", "FAILING") + .put(TYPE, "FAILING") .put("num_messages", 5) .build())); - final String connectionName = "test-connection"; final UUID sourceId = source.getSourceId(); final UUID destinationId = destination.getDestinationId(); final AirbyteCatalog catalog = testHarness.discoverSourceSchema(sourceId); final UUID connectionId = - testHarness.createConnection(connectionName, sourceId, destinationId, Collections.emptyList(), catalog, null) + testHarness.createConnection(TEST_CONNECTION, sourceId, destinationId, Collections.emptyList(), catalog, null) .getConnectionId(); final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java index 43ed913b2aad..f4c6e3b35b82 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java @@ -103,6 +103,8 @@ record DestinationCdcRecordMatcher(JsonNode sourceRecord, Instant minUpdatedAt, private static final String ID_AND_NAME_TABLE = "id_and_name"; private static final String COLOR_PALETTE_TABLE = "color_palette"; private static final String COLUMN_COLOR = "color"; + private static final String STARTING = "Starting {}"; + private static final String STARTING_SYNC_ONE = "Starting {} sync 1"; // version of the postgres destination connector that was built with the // old Airbyte protocol that does not contain any per-stream logic/fields @@ -156,10 +158,10 @@ void end() { @Test void testIncrementalCdcSync(final TestInfo testInfo) throws Exception { - LOGGER.info("Starting {}", testInfo.getDisplayName()); + LOGGER.info(STARTING, testInfo.getDisplayName()); final UUID connectionId = createCdcConnection(); - LOGGER.info("Starting {} sync 1", testInfo.getDisplayName()); + LOGGER.info(STARTING_SYNC_ONE, testInfo.getDisplayName()); final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); @@ -252,7 +254,7 @@ void testIncrementalCdcSync(final TestInfo testInfo) throws Exception { // built on the old protocol that did not have any per-stream state fields @Test void testIncrementalCdcSyncWithLegacyDestinationConnector(final TestInfo testInfo) throws Exception { - LOGGER.info("Starting {}", testInfo.getDisplayName()); + LOGGER.info(STARTING, testInfo.getDisplayName()); final UUID postgresDestDefId = testHarness.getPostgresDestinationDefinitionId(); // Fetch the current/most recent source definition version final DestinationDefinitionRead destinationDefinitionRead = apiClient.getDestinationDefinitionApi().getDestinationDefinition( @@ -273,10 +275,10 @@ void testIncrementalCdcSyncWithLegacyDestinationConnector(final TestInfo testInf @Test void testDeleteRecordCdcSync(final TestInfo testInfo) throws Exception { - LOGGER.info("Starting {}", testInfo.getDisplayName()); + LOGGER.info(STARTING, testInfo.getDisplayName()); final UUID connectionId = createCdcConnection(); - LOGGER.info("Starting {} sync 1", testInfo.getDisplayName()); + LOGGER.info(STARTING_SYNC_ONE, testInfo.getDisplayName()); final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); @@ -312,10 +314,10 @@ void testDeleteRecordCdcSync(final TestInfo testInfo) throws Exception { @Test void testPartialResetFromSchemaUpdate(final TestInfo testInfo) throws Exception { - LOGGER.info("Starting {}", testInfo.getDisplayName()); + LOGGER.info(STARTING, testInfo.getDisplayName()); final UUID connectionId = createCdcConnection(); - LOGGER.info("Starting {} sync 1", testInfo.getDisplayName()); + LOGGER.info(STARTING_SYNC_ONE, testInfo.getDisplayName()); final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); @@ -355,10 +357,10 @@ void testPartialResetFromSchemaUpdate(final TestInfo testInfo) throws Exception @Test void testPartialResetFromStreamSelection(final TestInfo testInfo) throws Exception { - LOGGER.info("Starting {}", testInfo.getDisplayName()); + LOGGER.info(STARTING, testInfo.getDisplayName()); final UUID connectionId = createCdcConnection(); - LOGGER.info("Starting {} sync 1", testInfo.getDisplayName()); + LOGGER.info(STARTING_SYNC_ONE, testInfo.getDisplayName()); final JobInfoRead connectionSyncRead1 = apiClient.getConnectionApi() .syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java index 8eb90b3b7865..38366d0a03a0 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java @@ -56,6 +56,8 @@ class ContainerOrchestratorAcceptanceTests { private static final Logger LOGGER = LoggerFactory.getLogger(ContainerOrchestratorAcceptanceTests.class); + private static final String AIRBYTE_WORKER = "airbyte-worker"; + private static final String DEFAULT = "default"; private static AirbyteAcceptanceTestHarness testHarness; private static AirbyteApiClient apiClient; @@ -119,10 +121,10 @@ void testDowntimeDuringSync() throws Exception { waitWhileJobHasStatus(apiClient.getJobsApi(), connectionSyncRead.getJob(), Set.of(JobStatus.PENDING)); LOGGER.info("Scaling down worker..."); - kubernetesClient.apps().deployments().inNamespace("default").withName("airbyte-worker").scale(0, true); + kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(0, true); LOGGER.info("Scaling up worker..."); - kubernetesClient.apps().deployments().inNamespace("default").withName("airbyte-worker").scale(1); + kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(1); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead.getJob()); @@ -156,8 +158,8 @@ void testCancelSyncWithInterruption() throws Exception { final JobInfoRead connectionSyncRead = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitWhileJobHasStatus(apiClient.getJobsApi(), connectionSyncRead.getJob(), Set.of(JobStatus.PENDING)); - kubernetesClient.apps().deployments().inNamespace("default").withName("airbyte-worker").scale(0, true); - kubernetesClient.apps().deployments().inNamespace("default").withName("airbyte-worker").scale(1); + kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(0, true); + kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(1); final var resp = apiClient.getJobsApi().cancelJob(new JobIdRequestBody().id(connectionSyncRead.getJob().getId())); assertEquals(JobStatus.CANCELLED, resp.getJob().getStatus()); @@ -190,7 +192,7 @@ void testCancelSyncWhenCancelledWhenWorkerIsNotRunning() throws Exception { Thread.sleep(1000); LOGGER.info("Scale down workers..."); - kubernetesClient.apps().deployments().inNamespace("default").withName("airbyte-worker").scale(0, true); + kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(0, true); LOGGER.info("Starting background cancellation request..."); final var pool = Executors.newSingleThreadExecutor(); @@ -210,7 +212,7 @@ void testCancelSyncWhenCancelledWhenWorkerIsNotRunning() throws Exception { Thread.sleep(2000); LOGGER.info("Scaling up workers..."); - kubernetesClient.apps().deployments().inNamespace("default").withName("airbyte-worker").scale(1); + kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(1); LOGGER.info("Waiting for cancellation to go into effect..."); assertEquals(JobStatus.CANCELLED, resp.get().getJob().getStatus()); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java index 0bfcdff57c56..c82d3785a7ef 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java @@ -104,6 +104,7 @@ import org.slf4j.MDC; @AllArgsConstructor +@SuppressWarnings("PMD.AvoidCatchingThrowable") public class WorkerApp { private static final Logger LOGGER = LoggerFactory.getLogger(WorkerApp.class); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java b/airbyte-workers/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java index 5c1315d22949..b4e360f3437f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java @@ -51,11 +51,12 @@ public static String getEntrypointEnvVariable(final ProcessFactory processFactor String outputLine = null; - String line; - while (((line = stdout.readLine()) != null) && outputLine == null) { + String line = stdout.readLine(); + while ((line != null) && outputLine == null) { if (line.contains("AIRBYTE_ENTRYPOINT")) { outputLine = line; } + line = stdout.readLine(); } process.waitFor(); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java index b8c1935a279a..405293663946 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java @@ -31,6 +31,8 @@ public class AirbyteIntegrationLauncher implements IntegrationLauncher { public static final String CHECK_JOB = "check"; public static final String DISCOVER_JOB = "discover"; + private static final String CONFIG = "--config"; + /** * A sync job can actually be broken down into the following steps. Try to be as precise as possible * with naming/labels to help operations. @@ -95,7 +97,7 @@ public Process check(final Path jobRoot, final String configFilename, final Stri getWorkerMetadata(), Collections.emptyMap(), "check", - "--config", configFilename); + CONFIG, configFilename); } @Override @@ -114,7 +116,7 @@ public Process discover(final Path jobRoot, final String configFilename, final S getWorkerMetadata(), Collections.emptyMap(), "discover", - "--config", configFilename); + CONFIG, configFilename); } @Override @@ -128,7 +130,7 @@ public Process read(final Path jobRoot, throws WorkerException { final List arguments = Lists.newArrayList( "read", - "--config", configFilename, + CONFIG, configFilename, "--catalog", catalogFilename); final Map files = new HashMap<>(); @@ -184,7 +186,7 @@ public Process write(final Path jobRoot, getWorkerMetadata(), Collections.emptyMap(), "write", - "--config", configFilename, + CONFIG, configFilename, "--catalog", catalogFilename); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java index 2a68780ed575..ba33953190b3 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java @@ -34,7 +34,7 @@ * outputs are passed to the selected worker. It also makes sures that the outputs of the worker are * persisted to the db. */ -@SuppressWarnings("PMD.UnusedFormalParameter") +@SuppressWarnings({"PMD.UnusedFormalParameter", "PMD.AvoidCatchingThrowable"}) public class TemporalAttemptExecution implements Supplier { private static final Logger LOGGER = LoggerFactory.getLogger(TemporalAttemptExecution.class); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/WorkerConfigsTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/WorkerConfigsTest.java index f43b80ec7962..bdf0c6c43905 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/WorkerConfigsTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/WorkerConfigsTest.java @@ -19,10 +19,11 @@ class WorkerConfigsTest { - private static final Map DEFAULT_NODE_SELECTORS = ImmutableMap.of("job", "default"); - private static final Map SPEC_NODE_SELECTORS = ImmutableMap.of("job", "spec"); - private static final Map CHECK_NODE_SELECTORS = ImmutableMap.of("job", "check"); - private static final Map DISCOVER_NODE_SELECTORS = ImmutableMap.of("job", "discover"); + private static final String JOB = "job"; + private static final Map DEFAULT_NODE_SELECTORS = ImmutableMap.of(JOB, "default"); + private static final Map SPEC_NODE_SELECTORS = ImmutableMap.of(JOB, "spec"); + private static final Map CHECK_NODE_SELECTORS = ImmutableMap.of(JOB, "check"); + private static final Map DISCOVER_NODE_SELECTORS = ImmutableMap.of(JOB, "discover"); private static final String DEFAULT_CPU_REQUEST = "0.1"; private static final String DEFAULT_CPU_LIMIT = "0.2"; private static final String DEFAULT_MEMORY_REQUEST = "100Mi"; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java index ac6780532266..6d8a5d890c30 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java @@ -86,6 +86,8 @@ class DefaultReplicationWorkerTest { private static final AirbyteMessage STATE_MESSAGE = AirbyteMessageUtils.createStateMessage("checkpoint", "1"); private static final AirbyteTraceMessage ERROR_TRACE_MESSAGE = AirbyteMessageUtils.createErrorTraceMessage("a connector error occurred", Double.valueOf(123)); + private static final String STREAM1 = "stream1"; + private static final String INDUCED_EXCEPTION = "induced exception"; private Path jobRoot; private AirbyteSource source; @@ -440,8 +442,8 @@ void testPopulatesOutputOnSuccess() throws WorkerException { when(messageTracker.getTotalRecordsEmitted()).thenReturn(12L); when(messageTracker.getTotalBytesEmitted()).thenReturn(100L); when(messageTracker.getTotalStateMessagesEmitted()).thenReturn(3L); - when(messageTracker.getStreamToEmittedBytes()).thenReturn(Collections.singletonMap("stream1", 100L)); - when(messageTracker.getStreamToEmittedRecords()).thenReturn(Collections.singletonMap("stream1", 12L)); + when(messageTracker.getStreamToEmittedBytes()).thenReturn(Collections.singletonMap(STREAM1, 100L)); + when(messageTracker.getStreamToEmittedRecords()).thenReturn(Collections.singletonMap(STREAM1, 12L)); final ReplicationWorker worker = new DefaultReplicationWorker( JOB_ID, @@ -466,7 +468,7 @@ void testPopulatesOutputOnSuccess() throws WorkerException { .withRecordsCommitted(12L)) // since success, should use emitted count .withStreamStats(Collections.singletonList( new StreamSyncStats() - .withStreamName("stream1") + .withStreamName(STREAM1) .withStats(new SyncStats() .withBytesEmitted(100L) .withRecordsEmitted(12L) @@ -494,7 +496,7 @@ void testPopulatesOutputOnSuccess() throws WorkerException { @Test void testPopulatesStateOnFailureIfAvailable() throws Exception { - doThrow(new IllegalStateException("induced exception")).when(source).close(); + doThrow(new IllegalStateException(INDUCED_EXCEPTION)).when(source).close(); when(messageTracker.getDestinationOutputState()).thenReturn(Optional.of(new State().withState(STATE_MESSAGE.getState().getData()))); final ReplicationWorker worker = new DefaultReplicationWorker( @@ -514,7 +516,7 @@ void testPopulatesStateOnFailureIfAvailable() throws Exception { @Test void testRetainsStateOnFailureIfNewStateNotAvailable() throws Exception { - doThrow(new IllegalStateException("induced exception")).when(source).close(); + doThrow(new IllegalStateException(INDUCED_EXCEPTION)).when(source).close(); final ReplicationWorker worker = new DefaultReplicationWorker( JOB_ID, @@ -534,14 +536,14 @@ void testRetainsStateOnFailureIfNewStateNotAvailable() throws Exception { @Test void testPopulatesStatsOnFailureIfAvailable() throws Exception { - doThrow(new IllegalStateException("induced exception")).when(source).close(); + doThrow(new IllegalStateException(INDUCED_EXCEPTION)).when(source).close(); when(messageTracker.getTotalRecordsEmitted()).thenReturn(12L); when(messageTracker.getTotalBytesEmitted()).thenReturn(100L); when(messageTracker.getTotalRecordsCommitted()).thenReturn(Optional.of(6L)); when(messageTracker.getTotalStateMessagesEmitted()).thenReturn(3L); - when(messageTracker.getStreamToEmittedBytes()).thenReturn(Collections.singletonMap("stream1", 100L)); - when(messageTracker.getStreamToEmittedRecords()).thenReturn(Collections.singletonMap("stream1", 12L)); - when(messageTracker.getStreamToCommittedRecords()).thenReturn(Optional.of(Collections.singletonMap("stream1", 6L))); + when(messageTracker.getStreamToEmittedBytes()).thenReturn(Collections.singletonMap(STREAM1, 100L)); + when(messageTracker.getStreamToEmittedRecords()).thenReturn(Collections.singletonMap(STREAM1, 12L)); + when(messageTracker.getStreamToCommittedRecords()).thenReturn(Optional.of(Collections.singletonMap(STREAM1, 6L))); final ReplicationWorker worker = new DefaultReplicationWorker( JOB_ID, @@ -561,7 +563,7 @@ void testPopulatesStatsOnFailureIfAvailable() throws Exception { .withRecordsCommitted(6L); final List expectedStreamStats = Collections.singletonList( new StreamSyncStats() - .withStreamName("stream1") + .withStreamName(STREAM1) .withStats(new SyncStats() .withBytesEmitted(100L) .withRecordsEmitted(12L) @@ -578,7 +580,7 @@ void testDoesNotPopulatesStateOnFailureIfNotAvailable() throws Exception { final StandardSyncInput syncInputWithoutState = Jsons.clone(syncInput); syncInputWithoutState.setState(null); - doThrow(new IllegalStateException("induced exception")).when(source).close(); + doThrow(new IllegalStateException(INDUCED_EXCEPTION)).when(source).close(); final ReplicationWorker worker = new DefaultReplicationWorker( JOB_ID, @@ -598,7 +600,7 @@ void testDoesNotPopulatesStateOnFailureIfNotAvailable() throws Exception { @Test void testDoesNotPopulateOnIrrecoverableFailure() { - doThrow(new IllegalStateException("induced exception")).when(messageTracker).getTotalRecordsEmitted(); + doThrow(new IllegalStateException(INDUCED_EXCEPTION)).when(messageTracker).getTotalRecordsEmitted(); final ReplicationWorker worker = new DefaultReplicationWorker( JOB_ID, diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java index 892be63cc865..68e3d304cdb6 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java @@ -29,6 +29,7 @@ class AirbyteMessageTrackerTest { private static final String STREAM_1 = "stream1"; private static final String STREAM_2 = "stream2"; private static final String STREAM_3 = "stream3"; + private static final String INDUCED_EXCEPTION = "induced exception"; private AirbyteMessageTracker messageTracker; @@ -182,7 +183,7 @@ void testGetCommittedRecordsByStream() { @Test void testGetCommittedRecordsByStream_emptyWhenAddStateThrowsException() throws Exception { - Mockito.doThrow(new StateDeltaTrackerException("induced exception")).when(mStateDeltaTracker).addState(Mockito.anyInt(), Mockito.anyMap()); + Mockito.doThrow(new StateDeltaTrackerException(INDUCED_EXCEPTION)).when(mStateDeltaTracker).addState(Mockito.anyInt(), Mockito.anyMap()); final AirbyteMessage r1 = AirbyteMessageUtils.createRecordMessage(STREAM_1, 1); final AirbyteMessage s1 = AirbyteMessageUtils.createStateMessage(1); @@ -196,7 +197,7 @@ void testGetCommittedRecordsByStream_emptyWhenAddStateThrowsException() throws E @Test void testGetCommittedRecordsByStream_emptyWhenCommitStateHashThrowsException() throws Exception { - Mockito.doThrow(new StateDeltaTrackerException("induced exception")).when(mStateDeltaTracker).commitStateHash(Mockito.anyInt()); + Mockito.doThrow(new StateDeltaTrackerException(INDUCED_EXCEPTION)).when(mStateDeltaTracker).commitStateHash(Mockito.anyInt()); final AirbyteMessage r1 = AirbyteMessageUtils.createRecordMessage(STREAM_1, 1); final AirbyteMessage s1 = AirbyteMessageUtils.createStateMessage(1); @@ -246,7 +247,7 @@ void testTotalRecordsCommitted() { @Test void testGetTotalRecordsCommitted_emptyWhenAddStateThrowsException() throws Exception { - Mockito.doThrow(new StateDeltaTrackerException("induced exception")).when(mStateDeltaTracker).addState(Mockito.anyInt(), Mockito.anyMap()); + Mockito.doThrow(new StateDeltaTrackerException(INDUCED_EXCEPTION)).when(mStateDeltaTracker).addState(Mockito.anyInt(), Mockito.anyMap()); final AirbyteMessage r1 = AirbyteMessageUtils.createRecordMessage(STREAM_1, 1); final AirbyteMessage s1 = AirbyteMessageUtils.createStateMessage(1); @@ -260,7 +261,7 @@ void testGetTotalRecordsCommitted_emptyWhenAddStateThrowsException() throws Exce @Test void testGetTotalRecordsCommitted_emptyWhenCommitStateHashThrowsException() throws Exception { - Mockito.doThrow(new StateDeltaTrackerException("induced exception")).when(mStateDeltaTracker).commitStateHash(Mockito.anyInt()); + Mockito.doThrow(new StateDeltaTrackerException(INDUCED_EXCEPTION)).when(mStateDeltaTracker).commitStateHash(Mockito.anyInt()); final AirbyteMessage r1 = AirbyteMessageUtils.createRecordMessage(STREAM_1, 1); final AirbyteMessage s1 = AirbyteMessageUtils.createStateMessage(1); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteProtocolPredicateTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteProtocolPredicateTest.java index 1f1b548a3e6e..9750e4fbf7b3 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteProtocolPredicateTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/internal/AirbyteProtocolPredicateTest.java @@ -15,6 +15,7 @@ class AirbyteProtocolPredicateTest { private static final String STREAM_NAME = "user_preferences"; private static final String FIELD_NAME = "favorite_color"; + private static final String GREEN = "green"; private AirbyteProtocolPredicate predicate; @@ -25,7 +26,7 @@ void setup() { @Test void testValid() { - assertTrue(predicate.test(Jsons.jsonNode(AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "green")))); + assertTrue(predicate.test(Jsons.jsonNode(AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, GREEN)))); } @Test @@ -36,7 +37,7 @@ void testInValid() { @Test void testConcatenatedValid() { final String concatenated = - Jsons.serialize(AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "green")) + Jsons.serialize(AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, GREEN)) + Jsons.serialize(AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "yellow")); assertTrue(predicate.test(Jsons.deserialize(concatenated))); @@ -45,7 +46,7 @@ void testConcatenatedValid() { @Test void testMissingNewLineAndLineStartsWithValidRecord() { final String concatenated = - Jsons.serialize(AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "green")) + Jsons.serialize(AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, GREEN)) + "{ \"fish\": \"tuna\"}"; assertTrue(predicate.test(Jsons.deserialize(concatenated))); @@ -55,7 +56,7 @@ void testMissingNewLineAndLineStartsWithValidRecord() { void testMissingNewLineAndLineStartsWithInvalidRecord() { final String concatenated = "{ \"fish\": \"tuna\"}" - + Jsons.serialize(AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "green")); + + Jsons.serialize(AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, GREEN)); assertFalse(predicate.test(Jsons.deserialize(concatenated))); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java index ee2207b513ad..8c19f199fc8a 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java @@ -21,6 +21,8 @@ class NamespacingMapperTest { private static final String OUTPUT_PREFIX = "output_"; private static final String STREAM_NAME = "user_preferences"; private static final String FIELD_NAME = "favorite_color"; + private static final String BLUE = "blue"; + private static final ConfiguredAirbyteCatalog CATALOG = CatalogHelpers.createConfiguredAirbyteCatalog( STREAM_NAME, INPUT_NAMESPACE, @@ -28,7 +30,7 @@ class NamespacingMapperTest { private static final AirbyteMessage RECORD_MESSAGE = createRecordMessage(); private static AirbyteMessage createRecordMessage() { - final AirbyteMessage message = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, "blue"); + final AirbyteMessage message = AirbyteMessageUtils.createRecordMessage(STREAM_NAME, FIELD_NAME, BLUE); message.getRecord().withNamespace(INPUT_NAMESPACE); return message; } @@ -48,7 +50,7 @@ void testSourceNamespace() { assertEquals(expectedCatalog, actualCatalog); final AirbyteMessage originalMessage = Jsons.clone(RECORD_MESSAGE); - final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, "blue"); + final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, BLUE); expectedMessage.getRecord().withNamespace(INPUT_NAMESPACE); final AirbyteMessage actualMessage = mapper.mapMessage(RECORD_MESSAGE); @@ -75,7 +77,7 @@ void testEmptySourceNamespace() { assertEquals(originalMessage, RECORD_MESSAGE); originalMessage.getRecord().withNamespace(null); - final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, "blue"); + final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, BLUE); expectedMessage.getRecord().withNamespace(null); final AirbyteMessage actualMessage = mapper.mapMessage(originalMessage); @@ -97,7 +99,7 @@ void testDestinationNamespace() { assertEquals(expectedCatalog, actualCatalog); final AirbyteMessage originalMessage = Jsons.clone(RECORD_MESSAGE); - final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, "blue"); + final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, BLUE); final AirbyteMessage actualMessage = mapper.mapMessage(RECORD_MESSAGE); assertEquals(originalMessage, RECORD_MESSAGE); @@ -119,7 +121,7 @@ void testCustomFormatWithVariableNamespace() { assertEquals(expectedCatalog, actualCatalog); final AirbyteMessage originalMessage = Jsons.clone(RECORD_MESSAGE); - final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, "blue"); + final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, BLUE); expectedMessage.getRecord().withNamespace(expectedNamespace); final AirbyteMessage actualMessage = mapper.mapMessage(RECORD_MESSAGE); @@ -142,7 +144,7 @@ void testCustomFormatWithoutVariableNamespace() { assertEquals(expectedCatalog, actualCatalog); final AirbyteMessage originalMessage = Jsons.clone(RECORD_MESSAGE); - final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, "blue"); + final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, BLUE); expectedMessage.getRecord().withNamespace(expectedNamespace); final AirbyteMessage actualMessage = mapper.mapMessage(RECORD_MESSAGE); @@ -169,7 +171,7 @@ void testEmptyCustomFormatWithVariableNamespace() { assertEquals(originalMessage, RECORD_MESSAGE); originalMessage.getRecord().withNamespace(null); - final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, "blue"); + final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage(OUTPUT_PREFIX + STREAM_NAME, FIELD_NAME, BLUE); expectedMessage.getRecord().withNamespace(null); final AirbyteMessage actualMessage = mapper.mapMessage(originalMessage); @@ -193,7 +195,7 @@ void testEmptyPrefix() { final AirbyteMessage originalMessage = Jsons.clone(RECORD_MESSAGE); final AirbyteMessage expectedMessage = AirbyteMessageUtils.createRecordMessage( STREAM_NAME, - FIELD_NAME, "blue"); + FIELD_NAME, BLUE); expectedMessage.getRecord().withNamespace(INPUT_NAMESPACE); final AirbyteMessage actualMessage = mapper.mapMessage(RECORD_MESSAGE); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java index 1b3ad9ac6f9d..03bf563792b3 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java @@ -33,18 +33,21 @@ @ExtendWith(MockitoExtension.class) class AirbyteIntegrationLauncherTest { + private static final String CONFIG = "config"; + private static final String CATALOG = "catalog"; + private static final String CONFIG_ARG = "--config"; private static final String JOB_ID = "0"; private static final int JOB_ATTEMPT = 0; private static final Path JOB_ROOT = Path.of("abc"); public static final String FAKE_IMAGE = "fake_image"; private static final Map CONFIG_FILES = ImmutableMap.of( - "config", "{}"); + CONFIG, "{}"); private static final Map CONFIG_CATALOG_FILES = ImmutableMap.of( - "config", "{}", - "catalog", "{}"); + CONFIG, "{}", + CATALOG, "{}"); private static final Map CONFIG_CATALOG_STATE_FILES = ImmutableMap.of( - "config", "{}", - "catalog", "{}", + CONFIG, "{}", + CATALOG, "{}", "state", "{}"); private static final Map JOB_METADATA = Map.of( WorkerEnvConstants.WORKER_CONNECTOR_IMAGE, FAKE_IMAGE, @@ -75,7 +78,7 @@ void spec() throws WorkerException { @Test void check() throws WorkerException { - launcher.check(JOB_ROOT, "config", "{}"); + launcher.check(JOB_ROOT, CONFIG, "{}"); Mockito.verify(processFactory).create(CHECK_JOB, JOB_ID, JOB_ATTEMPT, JOB_ROOT, FAKE_IMAGE, false, CONFIG_FILES, null, workerConfigs.getResourceRequirements(), @@ -83,12 +86,12 @@ void check() throws WorkerException { JOB_METADATA, Map.of(), "check", - "--config", "config"); + CONFIG_ARG, CONFIG); } @Test void discover() throws WorkerException { - launcher.discover(JOB_ROOT, "config", "{}"); + launcher.discover(JOB_ROOT, CONFIG, "{}"); Mockito.verify(processFactory).create(DISCOVER_JOB, JOB_ID, JOB_ATTEMPT, JOB_ROOT, FAKE_IMAGE, false, CONFIG_FILES, null, workerConfigs.getResourceRequirements(), @@ -96,12 +99,12 @@ void discover() throws WorkerException { JOB_METADATA, Map.of(), "discover", - "--config", "config"); + CONFIG_ARG, CONFIG); } @Test void read() throws WorkerException { - launcher.read(JOB_ROOT, "config", "{}", "catalog", "{}", "state", "{}"); + launcher.read(JOB_ROOT, CONFIG, "{}", CATALOG, "{}", "state", "{}"); Mockito.verify(processFactory).create(READ_STEP, JOB_ID, JOB_ATTEMPT, JOB_ROOT, FAKE_IMAGE, false, CONFIG_CATALOG_STATE_FILES, null, workerConfigs.getResourceRequirements(), @@ -110,14 +113,14 @@ void read() throws WorkerException { Map.of(), Lists.newArrayList( "read", - "--config", "config", - "--catalog", "catalog", + CONFIG_ARG, CONFIG, + "--catalog", CATALOG, "--state", "state").toArray(new String[0])); } @Test void write() throws WorkerException { - launcher.write(JOB_ROOT, "config", "{}", "catalog", "{}"); + launcher.write(JOB_ROOT, CONFIG, "{}", CATALOG, "{}"); Mockito.verify(processFactory).create(WRITE_STEP, JOB_ID, JOB_ATTEMPT, JOB_ROOT, FAKE_IMAGE, true, CONFIG_CATALOG_FILES, null, workerConfigs.getResourceRequirements(), @@ -125,8 +128,8 @@ void write() throws WorkerException { JOB_METADATA, Map.of(), "write", - "--config", "config", - "--catalog", "catalog"); + CONFIG_ARG, CONFIG, + "--catalog", CATALOG); } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java index f77325797215..340049eb57be 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java @@ -35,6 +35,8 @@ class DockerProcessFactoryTest { private static final Path TEST_ROOT = Path.of("/tmp/airbyte_tests"); + private static final String PROCESS_FACTORY = "process_factory"; + private static final String BUSYBOX = "busybox"; /** * {@link DockerProcessFactoryTest#testImageExists()} will fail if jq is not installed. The logs get @@ -62,15 +64,15 @@ void testJqExists() throws IOException { */ @Test void testImageExists() throws IOException, WorkerException { - final Path workspaceRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "process_factory"); + final Path workspaceRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), PROCESS_FACTORY); final DockerProcessFactory processFactory = new DockerProcessFactory(new WorkerConfigs(new EnvConfigs()), workspaceRoot, null, null, null); - assertTrue(processFactory.checkImageExists("busybox")); + assertTrue(processFactory.checkImageExists(BUSYBOX)); } @Test void testImageDoesNotExist() throws IOException, WorkerException { - final Path workspaceRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "process_factory"); + final Path workspaceRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), PROCESS_FACTORY); final DockerProcessFactory processFactory = new DockerProcessFactory(new WorkerConfigs(new EnvConfigs()), workspaceRoot, null, null, null); assertFalse(processFactory.checkImageExists("airbyte/fake:0.1.2")); @@ -78,12 +80,12 @@ void testImageDoesNotExist() throws IOException, WorkerException { @Test void testFileWriting() throws IOException, WorkerException { - final Path workspaceRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "process_factory"); + final Path workspaceRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), PROCESS_FACTORY); final Path jobRoot = workspaceRoot.resolve("job"); final DockerProcessFactory processFactory = new DockerProcessFactory(new WorkerConfigs(new EnvConfigs()), workspaceRoot, null, null, null); - processFactory.create("tester", "job_id", 0, jobRoot, "busybox", false, ImmutableMap.of("config.json", "{\"data\": 2}"), "echo hi", + processFactory.create("tester", "job_id", 0, jobRoot, BUSYBOX, false, ImmutableMap.of("config.json", "{\"data\": 2}"), "echo hi", new WorkerConfigs(new EnvConfigs()).getResourceRequirements(), Map.of(), Map.of(), Map.of()); assertEquals( @@ -96,7 +98,7 @@ void testFileWriting() throws IOException, WorkerException { */ @Test void testEnvMapSet() throws IOException, WorkerException, InterruptedException { - final Path workspaceRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), "process_factory"); + final Path workspaceRoot = Files.createTempDirectory(Files.createDirectories(TEST_ROOT), PROCESS_FACTORY); final Path jobRoot = workspaceRoot.resolve("job"); final WorkerConfigs workerConfigs = spy(new WorkerConfigs(new EnvConfigs())); @@ -117,7 +119,7 @@ void testEnvMapSet() throws IOException, WorkerException, InterruptedException { "job_id", 0, jobRoot, - "busybox", + BUSYBOX, false, Map.of(), "/bin/sh", @@ -149,7 +151,7 @@ private void waitForDockerToInitialize(final ProcessFactory processFactory, fina "job_id_" + RandomStringUtils.randomAlphabetic(4), 0, jobRoot, - "busybox", + BUSYBOX, false, Map.of(), "/bin/sh", diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java index b2e8564717b5..c39044fedc8f 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java @@ -90,6 +90,8 @@ class TemporalClientTest { .withDockerImage(IMAGE_NAME1); private static final String NAMESPACE = "namespace"; private static final StreamDescriptor STREAM_DESCRIPTOR = new StreamDescriptor().withName("name"); + private static final String UNCHECKED = "unchecked"; + private static final String EXCEPTION_MESSAGE = "Force state exception to simulate workflow not running"; private WorkflowClient workflowClient; private TemporalClient temporalClient; @@ -117,7 +119,7 @@ void setup() throws IOException { @DisplayName("Test execute method.") class ExecuteJob { - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @Test void testExecute() { final Supplier supplier = mock(Supplier.class); @@ -132,7 +134,7 @@ void testExecute() { assertEquals(logPath, response.getMetadata().getLogPath()); } - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @Test void testExecuteWithException() { final Supplier supplier = mock(Supplier.class); @@ -309,7 +311,7 @@ void migrateCalled() { class DeleteConnection { @Test - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @DisplayName("Test delete connection method when workflow is in a running state.") void testDeleteConnection() { final ConnectionManagerWorkflow mConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); @@ -336,12 +338,12 @@ void testDeleteConnection() { } @Test - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @DisplayName("Test delete connection method when workflow is in an unexpected state") void testDeleteConnectionInUnexpectedState() { final ConnectionManagerWorkflow mTerminatedConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); when(mTerminatedConnectionManagerWorkflow.getState()) - .thenThrow(new IllegalStateException("Force state exception to simulate workflow not running")); + .thenThrow(new IllegalStateException(EXCEPTION_MESSAGE)); when(workflowClient.newWorkflowStub(any(Class.class), any(String.class))).thenReturn(mTerminatedConnectionManagerWorkflow); final ConnectionManagerWorkflow mNewConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); @@ -363,7 +365,7 @@ void testDeleteConnectionInUnexpectedState() { } @Test - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @DisplayName("Test delete connection method when workflow has already been deleted") void testDeleteConnectionOnDeletedWorkflow() { final ConnectionManagerWorkflow mConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); @@ -386,7 +388,7 @@ void testDeleteConnectionOnDeletedWorkflow() { class UpdateConnection { @Test - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @DisplayName("Test update connection when workflow is running") void testUpdateConnection() { final ConnectionManagerWorkflow mConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); @@ -403,12 +405,12 @@ void testUpdateConnection() { } @Test - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @DisplayName("Test update connection method starts a new workflow when workflow is in an unexpected state") void testUpdateConnectionInUnexpectedState() { final ConnectionManagerWorkflow mConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); - when(mConnectionManagerWorkflow.getState()).thenThrow(new IllegalStateException("Force state exception to simulate workflow not running")); + when(mConnectionManagerWorkflow.getState()).thenThrow(new IllegalStateException(EXCEPTION_MESSAGE)); when(workflowClient.newWorkflowStub(any(Class.class), any(String.class))).thenReturn(mConnectionManagerWorkflow); doReturn(mConnectionManagerWorkflow).when(temporalClient).submitConnectionUpdaterAsync(CONNECTION_ID); @@ -425,7 +427,7 @@ void testUpdateConnectionInUnexpectedState() { } @Test - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @DisplayName("Test update connection method does nothing when connection is deleted") void testUpdateConnectionDeletedWorkflow() { final ConnectionManagerWorkflow mConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); @@ -491,7 +493,7 @@ void testStartNewManualSyncAlreadyRunning() { void testStartNewManualSyncRepairsBadWorkflowState() { final ConnectionManagerWorkflow mTerminatedConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); when(mTerminatedConnectionManagerWorkflow.getState()) - .thenThrow(new IllegalStateException("Force state exception to simulate workflow not running")); + .thenThrow(new IllegalStateException(EXCEPTION_MESSAGE)); when(mTerminatedConnectionManagerWorkflow.getJobInformation()).thenReturn(new JobInformation(JOB_ID, ATTEMPT_ID)); final ConnectionManagerWorkflow mNewConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); @@ -525,7 +527,7 @@ void testStartNewManualSyncRepairsBadWorkflowState() { } @Test - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @DisplayName("Test startNewManualSync returns a failure reason when connection is deleted") void testStartNewManualSyncDeletedWorkflow() { final ConnectionManagerWorkflow mConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); @@ -573,7 +575,7 @@ void testStartNewCancellationSuccess() { void testStartNewCancellationRepairsBadWorkflowState() { final ConnectionManagerWorkflow mTerminatedConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); when(mTerminatedConnectionManagerWorkflow.getState()) - .thenThrow(new IllegalStateException("Force state exception to simulate workflow not running")); + .thenThrow(new IllegalStateException(EXCEPTION_MESSAGE)); when(mTerminatedConnectionManagerWorkflow.getJobInformation()).thenReturn(new JobInformation(JOB_ID, ATTEMPT_ID)); final ConnectionManagerWorkflow mNewConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); @@ -606,7 +608,7 @@ void testStartNewCancellationRepairsBadWorkflowState() { } @Test - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @DisplayName("Test startNewCancellation returns a failure reason when connection is deleted") void testStartNewCancellationDeletedWorkflow() { final ConnectionManagerWorkflow mConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); @@ -665,7 +667,7 @@ void testResetConnectionSuccess() throws IOException { void testResetConnectionRepairsBadWorkflowState() throws IOException { final ConnectionManagerWorkflow mTerminatedConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); when(mTerminatedConnectionManagerWorkflow.getState()) - .thenThrow(new IllegalStateException("Force state exception to simulate workflow not running")); + .thenThrow(new IllegalStateException(EXCEPTION_MESSAGE)); when(mTerminatedConnectionManagerWorkflow.getJobInformation()).thenReturn(new JobInformation(JOB_ID, ATTEMPT_ID)); final ConnectionManagerWorkflow mNewConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); @@ -705,7 +707,7 @@ void testResetConnectionRepairsBadWorkflowState() throws IOException { } @Test - @SuppressWarnings("unchecked") + @SuppressWarnings(UNCHECKED) @DisplayName("Test resetConnection returns a failure reason when connection is deleted") void testResetConnectionDeletedWorkflow() throws IOException { final ConnectionManagerWorkflow mConnectionManagerWorkflow = mock(ConnectionManagerWorkflow.class); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalUtilsTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalUtilsTest.java index e7b55be8b863..37bdaa440515 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalUtilsTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalUtilsTest.java @@ -52,6 +52,7 @@ class TemporalUtilsTest { private static final String TASK_QUEUE = "default"; + private static final String BEFORE = "before: {}"; @Test void testAsyncExecute() throws Exception { @@ -314,13 +315,13 @@ public Activity1Impl(final VoidCallable callable) { @Override public void activity() { - LOGGER.info("before: {}", ACTIVITY1); + LOGGER.info(BEFORE, ACTIVITY1); try { callable.call(); } catch (final Exception e) { throw new RuntimeException(e); } - LOGGER.info("before: {}", ACTIVITY1); + LOGGER.info(BEFORE, ACTIVITY1); } } @@ -381,7 +382,7 @@ public Activity1Impl(final AtomicInteger timesReachedEnd) { @Override public void activity(final String arg) { - LOGGER.info("before: {}", ACTIVITY1); + LOGGER.info(BEFORE, ACTIVITY1); final ActivityExecutionContext context = Activity.getExecutionContext(); TemporalUtils.withBackgroundHeartbeat( new AtomicReference<>(null), @@ -401,7 +402,7 @@ public void activity(final String arg) { }, () -> context); timesReachedEnd.incrementAndGet(); - LOGGER.info("before: {}", ACTIVITY1); + LOGGER.info(BEFORE, ACTIVITY1); } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java index af981135fb9a..567e219159b3 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java @@ -109,6 +109,7 @@ class ConnectionManagerWorkflowTest { Mockito.mock(AutoDisableConnectionActivity.class, Mockito.withSettings().withoutAnnotations()); private static final StreamResetActivity mStreamResetActivity = Mockito.mock(StreamResetActivity.class, Mockito.withSettings().withoutAnnotations()); + private static final String EVENT = "event = "; private TestWorkflowEnvironment testEnv; private WorkflowClient client; @@ -568,7 +569,7 @@ void cancelRunning() throws InterruptedException { for (final ChangedStateEvent event : events) { if (event.isValue()) { - log.info("event = " + event); + log.info(EVENT + event); } } @@ -617,7 +618,7 @@ void deleteRunning() throws InterruptedException { for (final ChangedStateEvent event : events) { if (event.isValue()) { - log.info("event = " + event); + log.info(EVENT + event); } } @@ -697,7 +698,7 @@ void resetCancelRunningWorkflow() throws InterruptedException { for (final ChangedStateEvent event : events) { if (event.isValue()) { - log.info("event = " + event); + log.info(EVENT + event); } } @@ -777,7 +778,7 @@ void updatedSignalReceivedWhileRunning() throws InterruptedException { for (final ChangedStateEvent event : events) { if (event.isValue()) { - log.info("event = " + event); + log.info(EVENT + event); } } diff --git a/tools/gradle/pmd/rules.xml b/tools/gradle/pmd/rules.xml index e7358e8f6116..1d2244ca3d3c 100644 --- a/tools/gradle/pmd/rules.xml +++ b/tools/gradle/pmd/rules.xml @@ -101,12 +101,6 @@ - - - - - - @@ -128,13 +122,12 @@ - - - - - - - + + + + + +