Merge branch 'master' into grubberr/10384-source-github

airbytehq · Mar 1, 2022 · 5e93677 · 5e93677 · github-actions · Mar 1, 2022
2 parents a5b5eb6 + 4aa103a
commit 5e93677
Show file tree

Hide file tree

Showing 97 changed files with 1,642 additions and 1,323 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.35.39-alpha
+current_version = 0.35.42-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

diff --git a/.env b/.env
@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.35.39-alpha
+VERSION=0.35.42-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile
@@ -5,6 +5,6 @@ ENV APPLICATION airbyte-bootloader
 
 WORKDIR /app
 
-ADD bin/${APPLICATION}-0.35.39-alpha.tar /app
+ADD bin/${APPLICATION}-0.35.42-alpha.tar /app
 
-ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.39-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.42-alpha/bin/${APPLICATION}"]
diff --git a/airbyte-cdk/python/airbyte_cdk/logger.py b/airbyte-cdk/python/airbyte_cdk/logger.py
@@ -9,6 +9,7 @@
 from typing import List, Tuple
 
 from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage
+from deprecated import deprecated
 
 TRACE_LEVEL_NUM = 5
 
@@ -102,6 +103,7 @@ def log_by_prefix(msg: str, default_level: str) -> Tuple[int, str]:
     return log_level, rendered_message
 
 
+@deprecated(version="0.1.47", reason="Use logging.getLogger('airbyte') instead")
 class AirbyteLogger:
     def log(self, level, message):
         log_record = AirbyteLogMessage(level=level, message=message)

diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -37,7 +37,7 @@
 - name: Clickhouse
   destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48
   dockerRepository: airbyte/destination-clickhouse
-  dockerImageTag: 0.1.3
+  dockerImageTag: 0.1.4
   documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse
 - name: DynamoDB
   destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89
@@ -108,7 +108,7 @@
 - name: MS SQL Server
   destinationDefinitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf
   dockerRepository: airbyte/destination-mssql
-  dockerImageTag: 0.1.14
+  dockerImageTag: 0.1.15
   documentationUrl: https://docs.airbyte.io/integrations/destinations/mssql
   icon: mssql.svg
 - name: MeiliSearch
@@ -126,19 +126,19 @@
 - name: MySQL
   destinationDefinitionId: ca81ee7c-3163-4246-af40-094cc31e5e42
   dockerRepository: airbyte/destination-mysql
-  dockerImageTag: 0.1.17
+  dockerImageTag: 0.1.18
   documentationUrl: https://docs.airbyte.io/integrations/destinations/mysql
   icon: mysql.svg
 - name: Oracle
   destinationDefinitionId: 3986776d-2319-4de9-8af8-db14c0996e72
   dockerRepository: airbyte/destination-oracle
-  dockerImageTag: 0.1.13
+  dockerImageTag: 0.1.15
   documentationUrl: https://docs.airbyte.io/integrations/destinations/oracle
   icon: oracle.svg
 - name: Postgres
   destinationDefinitionId: 25c5221d-dce2-4163-ade9-739ef790f503
   dockerRepository: airbyte/destination-postgres
-  dockerImageTag: 0.3.14
+  dockerImageTag: 0.3.15
   documentationUrl: https://docs.airbyte.io/integrations/destinations/postgres
   icon: postgresql.svg
 - name: Pulsar
@@ -162,7 +162,7 @@
 - name: Redshift
   destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc
   dockerRepository: airbyte/destination-redshift
-  dockerImageTag: 0.3.26
+  dockerImageTag: 0.3.27
   documentationUrl: https://docs.airbyte.io/integrations/destinations/redshift
   icon: redshift.svg
 - name: Rockset
@@ -185,7 +185,7 @@
 - name: Snowflake
   destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
   dockerRepository: airbyte/destination-snowflake
-  dockerImageTag: 0.4.16
+  dockerImageTag: 0.4.17
   documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake
   icon: snowflake.svg
   resourceRequirements:
@@ -207,7 +207,7 @@
 - name: MariaDB ColumnStore
   destinationDefinitionId: 294a4790-429b-40ae-9516-49826b9702e1
   dockerRepository: airbyte/destination-mariadb-columnstore
-  dockerImageTag: 0.1.3
+  dockerImageTag: 0.1.4
   documentationUrl: https://docs.airbyte.io/integrations/destinations/mariadb-columnstore
   icon: mariadb.svg
 - name: Streamr
@@ -216,3 +216,9 @@
   dockerImageTag: 0.0.1
   documentationUrl: https://docs.airbyte.io/integrations/destinations/streamr
   icon: streamr.svg
+- name: Scylla
+  destinationDefinitionId: 3dc6f384-cd6b-4be3-ad16-a41450899bf0
+  dockerRepository: airbyte/destination-scylla
+  dockerImageTag: 0.1.1
+  documentationUrl: https://docs.airbyte.io/integrations/destinations/scylla
+  icon: scylla.svg
diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -667,7 +667,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-clickhouse:0.1.3"
+- dockerImage: "airbyte/destination-clickhouse:0.1.4"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse"
     connectionSpecification:
@@ -2077,7 +2077,7 @@
     supportsDBT: false
     supported_destination_sync_modes:
     - "append"
-- dockerImage: "airbyte/destination-mssql:0.1.14"
+- dockerImage: "airbyte/destination-mssql:0.1.15"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql"
     connectionSpecification:
@@ -2464,7 +2464,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-mysql:0.1.17"
+- dockerImage: "airbyte/destination-mysql:0.1.18"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/mysql"
     connectionSpecification:
@@ -2629,7 +2629,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-oracle:0.1.13"
+- dockerImage: "airbyte/destination-oracle:0.1.15"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/oracle"
     connectionSpecification:
@@ -2853,11 +2853,11 @@
                 order: 4
     supportsIncremental: true
     supportsNormalization: false
-    supportsDBT: true
+    supportsDBT: false
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-postgres:0.3.14"
+- dockerImage: "airbyte/destination-postgres:0.3.15"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/postgres"
     connectionSpecification:
@@ -3272,7 +3272,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-redshift:0.3.26"
+- dockerImage: "airbyte/destination-redshift:0.3.27"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/redshift"
     connectionSpecification:
@@ -3825,7 +3825,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-snowflake:0.4.16"
+- dockerImage: "airbyte/destination-snowflake:0.4.17"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake"
     connectionSpecification:
@@ -4078,7 +4078,7 @@
     - "overwrite"
     - "append"
     - "append_dedup"
-- dockerImage: "airbyte/destination-mariadb-columnstore:0.1.3"
+- dockerImage: "airbyte/destination-mariadb-columnstore:0.1.4"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/mariadb-columnstore"
     connectionSpecification:
@@ -4257,3 +4257,60 @@
     supported_destination_sync_modes:
     - "append"
     - "append_dedup"
+- dockerImage: "airbyte/destination-scylla:0.1.1"
+  spec:
+    documentationUrl: "https://docs.airbyte.io/integrations/destinations/scylla"
+    connectionSpecification:
+      $schema: "http://json-schema.org/draft-07/schema#"
+      title: "Scylla Destination Spec"
+      type: "object"
+      required:
+      - "keyspace"
+      - "username"
+      - "password"
+      - "address"
+      - "port"
+      additionalProperties: true
+      properties:
+        keyspace:
+          title: "Keyspace"
+          description: "Default Scylla keyspace to create data in."
+          type: "string"
+          order: 0
+        username:
+          title: "Username"
+          description: "Username to use to access Scylla."
+          type: "string"
+          order: 1
+        password:
+          title: "Password"
+          description: "Password associated with Scylla."
+          type: "string"
+          airbyte_secret: true
+          order: 2
+        address:
+          title: "Address"
+          description: "Address to connect to."
+          type: "string"
+          order: 3
+        port:
+          title: "Port"
+          description: "Port of Scylla."
+          type: "integer"
+          minimum: 0
+          maximum: 65536
+          default: 9042
+          order: 4
+        replication:
+          title: "Replication factor"
+          type: "integer"
+          description: "Indicates to how many nodes the data should be replicated\
+            \ to."
+          default: 1
+          order: 5
+    supportsIncremental: true
+    supportsNormalization: false
+    supportsDBT: false
+    supported_destination_sync_modes:
+    - "overwrite"
+    - "append"
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -829,7 +829,7 @@
 - name: Zendesk Support
   sourceDefinitionId: 79c1aa37-dae3-42ae-b333-d1c105477715
   dockerRepository: airbyte/source-zendesk-support
-  dockerImageTag: 0.1.12
+  dockerImageTag: 0.2.0
   documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-support
   icon: zendesk.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -8765,7 +8765,7 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-zendesk-support:0.1.12"
+- dockerImage: "airbyte/source-zendesk-support:0.2.0"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-support"
     connectionSpecification:

diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile
@@ -26,12 +26,12 @@ RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] htt
 RUN apt-get update && apt-get install -y kubectl
 
 ENV APPLICATION airbyte-container-orchestrator
-ENV AIRBYTE_ENTRYPOINT "/app/${APPLICATION}-0.35.39-alpha/bin/${APPLICATION}"
+ENV AIRBYTE_ENTRYPOINT "/app/${APPLICATION}-0.35.42-alpha/bin/${APPLICATION}"
 
 WORKDIR /app
 
 # Move orchestrator app
-ADD bin/${APPLICATION}-0.35.39-alpha.tar /app
+ADD bin/${APPLICATION}-0.35.42-alpha.tar /app
 
 # wait for upstream dependencies to become available before starting server
-ENTRYPOINT ["/bin/bash", "-c", "/app/${APPLICATION}-0.35.39-alpha/bin/${APPLICATION}"]
+ENTRYPOINT ["/bin/bash", "-c", "/app/${APPLICATION}-0.35.42-alpha/bin/${APPLICATION}"]
diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java b/airbyte-db/lib/src/main/java/io/airbyte/db/Databases.java
@@ -4,6 +4,7 @@
 
 package io.airbyte.db;
 
+import com.google.common.collect.Maps;
 import io.airbyte.commons.lang.Exceptions;
 import io.airbyte.db.bigquery.BigQueryDatabase;
 import io.airbyte.db.jdbc.DefaultJdbcDatabase;
@@ -15,7 +16,6 @@
 import io.airbyte.db.mongodb.MongoDatabase;
 import java.io.IOException;
 import java.util.Map;
-import java.util.Optional;
 import java.util.function.Function;
 import lombok.val;
 import org.apache.commons.dbcp2.BasicDataSource;
@@ -41,7 +41,7 @@ public static Database createPostgresDatabaseWithRetry(final String username,
       try {
         val infinity = Integer.MAX_VALUE;
         database = createPostgresDatabaseWithRetryTimeout(username, password, jdbcConnectionString, isDbReady, infinity);
-      } catch (IOException e) {
+      } catch (final IOException e) {
         // This should theoretically never happen since we set the timeout to be a very high number.
       }
     }
@@ -131,9 +131,9 @@ public static Database createDatabase(final String username,
                                         final String jdbcConnectionString,
                                         final String driverClassName,
                                         final SQLDialect dialect,
-                                        final String connectionProperties) {
+                                        final Map<String, String> connectionProperties) {
     final BasicDataSource connectionPool =
-        createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Optional.ofNullable(connectionProperties));
+        createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties);
 
     return new Database(connectionPool, dialect);
   }
@@ -159,7 +159,7 @@ public static JdbcDatabase createJdbcDatabase(final String username,
                                                 final String password,
                                                 final String jdbcConnectionString,
                                                 final String driverClassName,
-                                                final String connectionProperties) {
+                                                final Map<String, String> connectionProperties) {
     return createJdbcDatabase(username, password, jdbcConnectionString, driverClassName, connectionProperties,
         JdbcUtils.getDefaultSourceOperations());
   }
@@ -168,10 +168,10 @@ public static JdbcDatabase createJdbcDatabase(final String username,
                                                 final String password,
                                                 final String jdbcConnectionString,
                                                 final String driverClassName,
-                                                final String connectionProperties,
+                                                final Map<String, String> connectionProperties,
                                                 final JdbcCompatibleSourceOperations<?> sourceOperations) {
     final BasicDataSource connectionPool =
-        createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Optional.ofNullable(connectionProperties));
+        createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties);
 
     return new DefaultJdbcDatabase(connectionPool, sourceOperations);
   }
@@ -181,10 +181,10 @@ public static JdbcDatabase createStreamingJdbcDatabase(final String username,
                                                          final String jdbcConnectionString,
                                                          final String driverClassName,
                                                          final JdbcStreamingQueryConfiguration jdbcStreamingQuery,
-                                                         final String connectionProperties,
+                                                         final Map<String, String> connectionProperties,
                                                          final JdbcCompatibleSourceOperations<?> sourceOperations) {
     final BasicDataSource connectionPool =
-        createBasicDataSource(username, password, jdbcConnectionString, driverClassName, Optional.ofNullable(connectionProperties));
+        createBasicDataSource(username, password, jdbcConnectionString, driverClassName, connectionProperties);
 
     return new StreamingJdbcDatabase(connectionPool, sourceOperations, jdbcStreamingQuery);
   }
@@ -194,27 +194,7 @@ private static BasicDataSource createBasicDataSource(final String username,
                                                        final String jdbcConnectionString,
                                                        final String driverClassName) {
     return createBasicDataSource(username, password, jdbcConnectionString, driverClassName,
-        Optional.empty());
-  }
-
-  /**
-   * Prefer to use the method that takes in the connection properties as a map.
-   */
-  @Deprecated
-  private static BasicDataSource createBasicDataSource(final String username,
-                                                       final String password,
-                                                       final String jdbcConnectionString,
-                                                       final String driverClassName,
-                                                       final Optional<String> connectionProperties) {
-    final BasicDataSource connectionPool = new BasicDataSource();
-    connectionPool.setDriverClassName(driverClassName);
-    connectionPool.setUsername(username);
-    connectionPool.setPassword(password);
-    connectionPool.setInitialSize(0);
-    connectionPool.setMaxTotal(5);
-    connectionPool.setUrl(jdbcConnectionString);
-    connectionProperties.ifPresent(connectionPool::setConnectionProperties);
-    return connectionPool;
+        Maps.newHashMap());
   }
 
   public static BasicDataSource createBasicDataSource(final String username,
Name	Value	Name	Value	Name	Value
Quality Gate Status	ERROR	Duplicated Blocks	0	Coverage	25.7
Duplicated Lines (%)	0.0	Vulnerabilities	0	Reliability Rating	A
Lines to Cover	74	Lines of Code	792	Bugs	0
Security Rating	A	Code Smells	4	Blocker Issues	0
Critical Issues	6	Major Issues	6	Minor Issues	6
Rule	File	Description	Message
python:S2638 (CRITICAL)	source_github/streams.py	Method overrides should not change contracts	Make parameter stream_state keyword-or-positional.
python:S2638 (CRITICAL)	source_github/streams.py	Method overrides should not change contracts	Make parameter stream_slice keyword-or-positional.
python:S2638 (CRITICAL)	source_github/streams.py	Method overrides should not change contracts	Make parameter next_page_token keyword-or-positional.
python:black_need_format (MINOR)	source_github/source.py	Please run one of the commands: "black --config ./pyproject.toml <path_to_updated_folder>" or "./gradlew format"	1 code part(s) should be updated.
flake8:E501 (MAJOR)	source_github/streams.py	line too long (82 > 79 characters)	line too long (143 > 140 characters)
python:isort_need_format (MINOR)	unit_tests/test_stream.py	Please run one of the commands: "isort <path_to_updated_folder>" or "./gradlew format"	1 code part(s) should be updated.
python:S3776 (CRITICAL)	source_github/streams.py:91	Cognitive Complexity of functions should not be too high	Refactor this function to reduce its Cognitive Complexity from 25 to the 15 allowed.
flake8:E501 (MAJOR)	source_github/streams.py	line too long (82 > 79 characters)	line too long (150 > 140 characters)
python:S5886 (MAJOR)	source_github/streams.py:84	Function return types should be consistent with their type hint	Return a value of type "Union[int, float]" instead of "NoneType" or update function "backoff_time" type hint.
python:S2638 (CRITICAL)	source_github/streams.py	Method overrides should not change contracts	Add missing parameters organization project_id.
python:isort_need_format (MINOR)	unit_tests/test_source.py	Please run one of the commands: "isort <path_to_updated_folder>" or "./gradlew format"	1 code part(s) should be updated.
python:isort_need_format (MINOR)	unit_tests/unit_test.py	Please run one of the commands: "isort <path_to_updated_folder>" or "./gradlew format"	1 code part(s) should be updated.
python:isort_need_format (MINOR)	main.py	Please run one of the commands: "isort <path_to_updated_folder>" or "./gradlew format"	1 code part(s) should be updated.
python:mypy_unknown (MINOR)	main.py	Unknown error	Duplicate module named "main" (also at "./fixtures/main.py")
flake8:E501 (MAJOR)	fixtures/github.py	line too long (82 > 79 characters)	line too long (151 > 140 characters)
python:S5797 (CRITICAL)	fixtures/github.py:79	Constants should not be used as conditions	Replace this expression; used as a condition it will always be constant.
python:S112 (MAJOR)	source_github/source.py:71	"Exception" and "BaseException" should not be raised	Replace this generic exception class with a more specific one.
flake8:C901 (MAJOR)	source_github/streams.py	flake8:C901	'GithubStream.read_records' is too complex (14)
File	Coverage	File	Coverage
fixtures/github.py	0.0	fixtures/main.py	0.0
source_github/init.py	100.0	source_github/source.py	48.9
source_github/streams.py	53.4