🎉 Jdbc sources: publish new version with adaptive fetch size (#12480)

* Default scaffold to use adaptive streaming config * Switch more connectors to use adaptive streaming config * Bump version for cockroach db * Bump version for db2 * Bump mssql version * Bump mysql version * Bump oracle version * Bump postgres version * Bump redshift version * Bump snowflake version * Bump tidb version * auto-bump connector version * Fix db2 findbug issue * auto-bump connector version * auto-bump connector version * auto-bump connector version * auto-bump connector version * Fix more findbug issues * auto-bump connector version * auto-bump connector version * auto-bump connector version * Fix findbug issue for mysql-strict-encrypt * Fix findbugs issue for oracle source * auto-bump connector version * Remove suppress warnings annotation * Fix oracle encrypt tests * Fix oracle encrypt acceptance test Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Apr 30, 2022 · 35f2aa9 · 35f2aa9
1 parent f6791b8
commit 35f2aa9
Show file tree

Hide file tree

Showing 46 changed files with 155 additions and 112 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -147,7 +147,7 @@
 - name: Cockroachdb
   sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003
   dockerRepository: airbyte/source-cockroachdb
-  dockerImageTag: 0.1.11
+  dockerImageTag: 0.1.12
   documentationUrl: https://docs.airbyte.io/integrations/sources/cockroachdb
   icon: cockroachdb.svg
   sourceType: database
@@ -348,7 +348,7 @@
 - name: IBM Db2
   sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2
   dockerRepository: airbyte/source-db2
-  dockerImageTag: 0.1.9
+  dockerImageTag: 0.1.10
   documentationUrl: https://docs.airbyte.io/integrations/sources/db2
   icon: db2.svg
   sourceType: database
@@ -459,7 +459,7 @@
 - name: Microsoft SQL Server (MSSQL)
   sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1
   dockerRepository: airbyte/source-mssql
-  dockerImageTag: 0.3.21
+  dockerImageTag: 0.3.22
   documentationUrl: https://docs.airbyte.io/integrations/sources/mssql
   icon: mssql.svg
   sourceType: database
@@ -501,7 +501,7 @@
 - name: MySQL
   sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
   dockerRepository: airbyte/source-mysql
-  dockerImageTag: 0.5.9
+  dockerImageTag: 0.5.10
   documentationUrl: https://docs.airbyte.io/integrations/sources/mysql
   icon: mysql.svg
   sourceType: database
@@ -535,7 +535,7 @@
 - name: Oracle DB
   sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83
   dockerRepository: airbyte/source-oracle
-  dockerImageTag: 0.3.14
+  dockerImageTag: 0.3.15
   documentationUrl: https://docs.airbyte.io/integrations/sources/oracle
   icon: oracle.svg
   sourceType: database
@@ -625,7 +625,7 @@
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 0.4.11
+  dockerImageTag: 0.4.12
   documentationUrl: https://docs.airbyte.io/integrations/sources/postgres
   icon: postgresql.svg
   sourceType: database
@@ -667,7 +667,7 @@
 - name: Redshift
   sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6
   dockerRepository: airbyte/source-redshift
-  dockerImageTag: 0.3.9
+  dockerImageTag: 0.3.10
   documentationUrl: https://docs.airbyte.io/integrations/sources/redshift
   icon: redshift.svg
   sourceType: database
@@ -751,7 +751,7 @@
 - name: Snowflake
   sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2
   dockerRepository: airbyte/source-snowflake
-  dockerImageTag: 0.1.11
+  dockerImageTag: 0.1.12
   documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake
   icon: snowflake.svg
   sourceType: database
@@ -792,7 +792,7 @@
 - name: TiDB
   sourceDefinitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae
   dockerRepository: airbyte/source-tidb
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.1
   documentationUrl: https://docs.airbyte.io/integrations/sources/tidb
   icon: tidb.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -1305,7 +1305,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-cockroachdb:0.1.11"
+- dockerImage: "airbyte/source-cockroachdb:0.1.12"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/cockroachdb"
     connectionSpecification:
@@ -3578,7 +3578,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "refresh_token"
-- dockerImage: "airbyte/source-db2:0.1.9"
+- dockerImage: "airbyte/source-db2:0.1.10"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/db2"
     connectionSpecification:
@@ -4669,7 +4669,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mssql:0.3.21"
+- dockerImage: "airbyte/source-mssql:0.3.22"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql"
     connectionSpecification:
@@ -4687,6 +4687,7 @@
           description: "The hostname of the database."
           title: "Host"
           type: "string"
+          order: 0
         port:
           description: "The port of the database."
           title: "Port"
@@ -4695,21 +4696,32 @@
           maximum: 65536
           examples:
           - "1433"
+          order: 1
         database:
           description: "The name of the database."
           title: "Database"
           type: "string"
           examples:
           - "master"
+          order: 2
         username:
           description: "The username which is used to access the database."
           title: "Username"
           type: "string"
+          order: 3
         password:
           description: "The password associated with the username."
           title: "Password"
           type: "string"
           airbyte_secret: true
+          order: 4
+        jdbc_url_params:
+          title: "JDBC URL Params"
+          description: "Additional properties to pass to the JDBC URL string when\
+            \ connecting to the database formatted as 'key=value' pairs separated\
+            \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
+          type: "string"
+          order: 5
         ssl_method:
           title: "SSL Method"
           type: "object"
@@ -4774,6 +4786,7 @@
           enum:
           - "STANDARD"
           - "CDC"
+          order: 8
         tunnel_method:
           type: "object"
           title: "SSH Tunnel Method"
@@ -5368,7 +5381,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mysql:0.5.9"
+- dockerImage: "airbyte/source-mysql:0.5.10"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql"
     connectionSpecification:
@@ -5757,7 +5770,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-oracle:0.3.14"
+- dockerImage: "airbyte/source-oracle:0.3.15"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/oracle"
     connectionSpecification:
@@ -6478,7 +6491,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-postgres:0.4.11"
+- dockerImage: "airbyte/source-postgres:0.4.12"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
@@ -6914,7 +6927,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-redshift:0.3.9"
+- dockerImage: "airbyte/source-redshift:0.3.10"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/redshift"
     connectionSpecification:
@@ -7972,7 +7985,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "refresh_token"
-- dockerImage: "airbyte/source-snowflake:0.1.11"
+- dockerImage: "airbyte/source-snowflake:0.1.12"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/snowflake"
     connectionSpecification:
@@ -8493,7 +8506,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-tidb:0.1.0"
+- dockerImage: "airbyte/source-tidb:0.1.1"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/tidb"
     connectionSpecification:

diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java
@@ -29,17 +29,25 @@ public static String getFullyQualifiedTableName(final String schemaName, final S
   }
 
   public static Map<String, String> parseJdbcParameters(final JsonNode config, final String jdbcUrlParamsKey) {
+    return parseJdbcParameters(config, jdbcUrlParamsKey, "&");
+  }
+
+  public static Map<String, String> parseJdbcParameters(final JsonNode config, final String jdbcUrlParamsKey, final String delimiter) {
     if (config.has(jdbcUrlParamsKey)) {
-      return parseJdbcParameters(config.get(jdbcUrlParamsKey).asText());
+      return parseJdbcParameters(config.get(jdbcUrlParamsKey).asText(), delimiter);
     } else {
       return Maps.newHashMap();
     }
   }
 
   public static Map<String, String> parseJdbcParameters(final String jdbcPropertiesString) {
+    return parseJdbcParameters(jdbcPropertiesString, "&");
+  }
+
+  public static Map<String, String> parseJdbcParameters(final String jdbcPropertiesString, final String delimiter) {
     final Map<String, String> parameters = new HashMap<>();
     if (!jdbcPropertiesString.isBlank()) {
-      final String[] keyValuePairs = jdbcPropertiesString.split("&");
+      final String[] keyValuePairs = jdbcPropertiesString.split(delimiter);
       for (final String kv : keyValuePairs) {
         final String[] split = kv.split("=");
         if (split.length == 2) {

diff --git a/...java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs b/...java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs
@@ -6,7 +6,7 @@ package io.airbyte.integrations.source.{{snakeCase name}};
 
 import com.fasterxml.jackson.databind.JsonNode;
 import io.airbyte.db.jdbc.JdbcUtils;
-import io.airbyte.db.jdbc.streaming.NoOpStreamingQueryConfig;
+import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig;
 import io.airbyte.integrations.base.IntegrationRunner;
 import io.airbyte.integrations.base.Source;
 import io.airbyte.integrations.source.jdbc.AbstractJdbcSource;
@@ -23,9 +23,9 @@ public class {{pascalCase name}}Source extends AbstractJdbcSource<JDBCType> impl
   static final String DRIVER_CLASS = "driver_name_here";
 
   public {{pascalCase name}}Source() {
-    // By default, NoOpStreamingQueryConfig class is used. If the JDBC supports custom
-    // fetch size, change it to AdaptiveStreamingQueryConfig for better performance.
-    super(DRIVER_CLASS, NoOpStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations());
+    // TODO: if the JDBC driver does not support custom fetch size, use NoOpStreamingQueryConfig
+    // instead of AdaptiveStreamingQueryConfig.
+    super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations());
   }
 
   // TODO The config is based on spec.json, update according to your DB

diff --git a/...tions/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/...tions/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java
@@ -178,8 +178,7 @@ public void testEncryption() throws SQLException {
             config.get("sid").asText()),
         "oracle.jdbc.driver.OracleDriver",
         JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" +
-            "oracle.net.encryption_types_client=( "
-            + algorithm + " )"));
+            "oracle.net.encryption_types_client=( " + algorithm + " )", ";"));
 
     final String network_service_banner =
         "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)";
@@ -204,8 +203,7 @@ public void testCheckProtocol() throws SQLException {
             clone.get("sid").asText()),
         "oracle.jdbc.driver.OracleDriver",
         JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" +
-            "oracle.net.encryption_types_client=( "
-            + algorithm + " )"));
+            "oracle.net.encryption_types_client=( " + algorithm + " )", ";"));
 
     final String network_service_banner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual";
     final List<JsonNode> collect = database.unsafeQuery(network_service_banner).collect(Collectors.toList());

diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.8
+LABEL io.airbyte.version=0.1.12
 LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt
diff --git a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.11
+LABEL io.airbyte.version=0.1.12
 LABEL io.airbyte.name=airbyte/source-cockroachdb
diff --git a/...ckroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java b/...ckroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java
@@ -12,7 +12,7 @@
 import io.airbyte.db.Databases;
 import io.airbyte.db.jdbc.JdbcDatabase;
 import io.airbyte.db.jdbc.JdbcUtils;
-import io.airbyte.db.jdbc.streaming.NoOpStreamingQueryConfig;
+import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig;
 import io.airbyte.integrations.base.IntegrationRunner;
 import io.airbyte.integrations.base.Source;
 import io.airbyte.integrations.base.ssh.SshWrappedSource;
@@ -41,7 +41,7 @@ public class CockroachDbSource extends AbstractJdbcSource<JDBCType> {
   public static final List<String> PORT_KEY = List.of("port");
 
   public CockroachDbSource() {
-    super(DRIVER_CLASS, NoOpStreamingQueryConfig::new, new CockroachJdbcSourceOperations());
+    super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, new CockroachJdbcSourceOperations());
   }
 
   public static Source sshWrappedSource() {

diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-db2-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.6
+LABEL io.airbyte.version=0.1.10
 LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt
diff --git a/...io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java b/...io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java
@@ -25,6 +25,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.concurrent.TimeUnit;
 import org.testcontainers.containers.Db2Container;
@@ -181,7 +182,7 @@ private String getCertificate() throws IOException, InterruptedException {
 
   private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException {
     final Runtime run = Runtime.getRuntime();
-    try (final PrintWriter out = new PrintWriter("certificate.pem")) {
+    try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) {
       out.print(certificate);
     }
     runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run);

diff --git a/...t/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java b/...t/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java
@@ -19,6 +19,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
 import java.sql.JDBCType;
 import java.util.Collections;
 import java.util.Set;
@@ -186,7 +187,7 @@ private static String getCertificate() throws IOException, InterruptedException
 
   private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException {
     final Runtime run = Runtime.getRuntime();
-    try (final PrintWriter out = new PrintWriter("certificate.pem")) {
+    try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) {
       out.print(certificate);
     }
     runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run);

diff --git a/airbyte-integrations/connectors/source-db2/Dockerfile b/airbyte-integrations/connectors/source-db2/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-db2
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.9
+LABEL io.airbyte.version=0.1.10
 LABEL io.airbyte.name=airbyte/source-db2
diff --git a/...ons/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java b/...ons/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java
@@ -16,6 +16,7 @@
 import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
 import java.sql.Connection;
 import java.sql.JDBCType;
 import java.sql.PreparedStatement;
@@ -143,7 +144,7 @@ private static String getKeyStorePassword(final JsonNode encryptionKeyStorePassw
   private static void convertAndImportCertificate(final String certificate, final String keyStorePassword)
       throws IOException, InterruptedException {
     final Runtime run = Runtime.getRuntime();
-    try (final PrintWriter out = new PrintWriter("certificate.pem")) {
+    try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) {
       out.print(certificate);
     }
     runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run);

diff --git a/...integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java b/...integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java
@@ -25,6 +25,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.concurrent.TimeUnit;
 import org.testcontainers.containers.Db2Container;
@@ -175,7 +176,7 @@ private String getCertificate() throws IOException, InterruptedException {
 
   private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException {
     final Runtime run = Runtime.getRuntime();
-    try (final PrintWriter out = new PrintWriter("certificate.pem")) {
+    try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) {
       out.print(certificate);
     }
     runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run);