🎉 Source Snowflake: support oauth (#10953)

* add oauth flow to source-snowflake * Add unit test for oauth flow * add docs to method * format code * change configs * fixed remarks * fixed comments * fixed PR remark * update with master * format code * fix PR remmakrs * add test for backward compatibility * bump version * small fix for test * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Apr 27, 2022 · 45a212c · 45a212c
1 parent 1413aca
commit 45a212c
Show file tree

Hide file tree

Showing 19 changed files with 1,014 additions and 102 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -751,7 +751,7 @@
 - name: Snowflake
   sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2
   dockerRepository: airbyte/source-snowflake
-  dockerImageTag: 0.1.10
+  dockerImageTag: 0.1.11
   documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake
   icon: snowflake.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -7955,7 +7955,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "refresh_token"
-- dockerImage: "airbyte/source-snowflake:0.1.10"
+- dockerImage: "airbyte/source-snowflake:0.1.11"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/snowflake"
     connectionSpecification:
@@ -7968,69 +7968,174 @@
       - "warehouse"
       - "database"
       - "schema"
-      - "username"
-      - "password"
-      additionalProperties: false
+      additionalProperties: true
       properties:
+        credentials:
+          title: "Authorization Method"
+          type: "object"
+          oneOf:
+          - type: "object"
+            title: "OAuth2.0"
+            order: 0
+            required:
+            - "client_id"
+            - "client_secret"
+            - "auth_type"
+            properties:
+              auth_type:
+                type: "string"
+                const: "OAuth"
+                default: "OAuth"
+                order: 0
+              client_id:
+                type: "string"
+                title: "Client ID"
+                description: "The Client ID of your Snowflake developer application."
+                airbyte_secret: true
+                order: 1
+              client_secret:
+                type: "string"
+                title: "Client Secret"
+                description: "The Client Secret of your Snowflake developer application."
+                airbyte_secret: true
+                order: 2
+              access_token:
+                type: "string"
+                title: "Access Token"
+                description: "Access Token for making authenticated requests."
+                airbyte_secret: true
+                order: 3
+              refresh_token:
+                type: "string"
+                title: "Refresh Token"
+                description: "Refresh Token for making authenticated requests."
+                airbyte_secret: true
+                order: 4
+          - title: "Username and Password"
+            type: "object"
+            required:
+            - "username"
+            - "password"
+            - "auth_type"
+            order: 1
+            properties:
+              auth_type:
+                type: "string"
+                const: "username/password"
+                default: "username/password"
+                order: 0
+              username:
+                description: "The username you created to allow Airbyte to access\
+                  \ the database."
+                examples:
+                - "AIRBYTE_USER"
+                type: "string"
+                title: "Username"
+                order: 1
+              password:
+                description: "The password associated with the username."
+                type: "string"
+                airbyte_secret: true
+                title: "Password"
+                order: 2
+          order: 0
         host:
           description: "The host domain of the snowflake instance (must include the\
             \ account, region, cloud environment, and end with snowflakecomputing.com)."
           examples:
           - "accountname.us-east-2.aws.snowflakecomputing.com"
           type: "string"
           title: "Account Name"
-          order: 0
+          order: 1
         role:
           description: "The role you created for Airbyte to access Snowflake."
           examples:
           - "AIRBYTE_ROLE"
           type: "string"
           title: "Role"
-          order: 1
+          order: 2
         warehouse:
           description: "The warehouse you created for Airbyte to access data."
           examples:
           - "AIRBYTE_WAREHOUSE"
           type: "string"
           title: "Warehouse"
-          order: 2
+          order: 3
         database:
           description: "The database you created for Airbyte to access data."
           examples:
           - "AIRBYTE_DATABASE"
           type: "string"
           title: "Database"
-          order: 3
+          order: 4
         schema:
           description: "The source Snowflake schema tables."
           examples:
           - "AIRBYTE_SCHEMA"
           type: "string"
           title: "Schema"
-          order: 4
-        username:
-          description: "The username you created to allow Airbyte to access the database."
-          examples:
-          - "AIRBYTE_USER"
-          type: "string"
-          title: "Username"
           order: 5
-        password:
-          description: "The password associated with the username."
-          type: "string"
-          airbyte_secret: true
-          title: "Password"
-          order: 6
         jdbc_url_params:
           description: "Additional properties to pass to the JDBC URL string when\
             \ connecting to the database formatted as 'key=value' pairs separated\
             \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
           title: "JDBC URL Params"
           type: "string"
-          order: 7
+          order: 6
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
+    advanced_auth:
+      auth_flow_type: "oauth2.0"
+      predicate_key:
+      - "credentials"
+      - "auth_type"
+      predicate_value: "OAuth"
+      oauth_config_specification:
+        oauth_user_input_from_connector_config_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            host:
+              type: "string"
+              path_in_connector_config:
+              - "host"
+        complete_oauth_output_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            access_token:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "access_token"
+            refresh_token:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "refresh_token"
+        complete_oauth_server_input_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            client_id:
+              type: "string"
+            client_secret:
+              type: "string"
+        complete_oauth_server_output_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            client_id:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "client_id"
+            client_secret:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "client_secret"
 - dockerImage: "airbyte/source-square:0.1.4"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/square"

diff --git a/...rc/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java b/...rc/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java
@@ -140,6 +140,13 @@ protected StandardCheckConnectionOutput runCheck() throws Exception {
             .run(new StandardCheckConnectionInput().withConnectionConfiguration(getConfig()), jobRoot);
   }
 
+  protected String runCheckAndGetStatusAsString(JsonNode config) throws Exception {
+    return new DefaultCheckConnectionWorker(
+        workerConfigs,
+        new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, workerConfigs.getResourceRequirements()))
+            .run(new StandardCheckConnectionInput().withConnectionConfiguration(config), jobRoot).getStatus().toString();
+  }
+
   protected AirbyteCatalog runDiscover() throws Exception {
     return new DefaultDiscoverCatalogWorker(
         workerConfigs,

diff --git a/...ors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/...ors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java
@@ -289,7 +289,7 @@ public JdbcDatabase createDatabase(final JsonNode config) throws SQLException {
     final JsonNode jdbcConfig = toDatabaseConfig(config);
 
     final JdbcDatabase database = Databases.createStreamingJdbcDatabase(
-        jdbcConfig.get("username").asText(),
+        jdbcConfig.has("username") ? jdbcConfig.get("username").asText() : null,
         jdbcConfig.has("password") ? jdbcConfig.get("password").asText() : null,
         jdbcConfig.get("jdbc_url").asText(),
         driverClass,

diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.10
+LABEL io.airbyte.version=0.1.11
 LABEL io.airbyte.name=airbyte/source-snowflake
diff --git a/airbyte-integrations/connectors/source-snowflake/README.md b/airbyte-integrations/connectors/source-snowflake/README.md
@@ -13,10 +13,28 @@
   "warehouse": "AIRBYTE_WAREHOUSE",
   "database": "AIRBYTE_DATABASE",
   "schema": "AIRBYTE_SCHEMA",
-  "username": "AIRBYTE_USER",
-  "password": "SOMEPASSWORD"
+  "credentails" {
+    "auth_type": "username/password",
+    "username": "AIRBYTE_USER",
+    "password": "SOMEPASSWORD"
+  }
+}
+```
+3. Create a file at `secrets/config_auth.json` with the following format:
+```
+{
+  "host": "ACCOUNT.REGION.PROVIDER.snowflakecomputing.com",
+  "role": "AIRBYTE_ROLE",
+  "warehouse": "AIRBYTE_WAREHOUSE",
+  "database": "AIRBYTE_DATABASE",
+  "schema": "AIRBYTE_SCHEMA",
+  "credentails" {
+    "auth_type": "OAuth",
+    "client_id": "client_id",
+    "client_secret": "client_secret",
+    "refresh_token": "refresh_token"
+  }
 }
 ```
-
 ## For Airbyte employees
 Put the contents of the `Snowflake Insert Test Creds` secret on Lastpass into `secrets/config.json` to be able to run integration tests locally.
diff --git a/airbyte-integrations/connectors/source-snowflake/build.gradle b/airbyte-integrations/connectors/source-snowflake/build.gradle
@@ -17,6 +17,7 @@ dependencies {
     implementation project(':airbyte-protocol:models')
     implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
     implementation group: 'net.snowflake', name: 'snowflake-jdbc', version: '3.13.9'
+    implementation 'com.zaxxer:HikariCP:5.0.1'
 
     testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc'))
     testImplementation project(':airbyte-test-utils')