Snowflake destination: support key pair authentication (#14388)

* Snowflake destination: support key pair authentication. * Snowflake destination: update docs * Snowflake destination: support key pair authentication for normalization, added tests * Snowflake destination: update normalization * Snowflake destination: update way of read secrets for test * Snowflake destination: moved test to another class for test purpose * Snowflake destination: update secrets for test purpose * Snowflake destination: revert changes added for test purpose * Snowflake destination: changes added for test purpose * Snowflake destination: updated required fields in specs * Snowflake destination: clean up * Snowflake destination: support encrypted key pair authentication (#14589) * Snowflake destination: add support for ecrypted private key * Snowflake destination: temp reverting for test purpose * Revert "Snowflake destination: temp reverting for test purpose" This reverts commit 260ecce. * Snowflake destination: add passphrase and remove auth_type from required fields * Snowflake destination: format code * Snowflake destination: clean up * Snowflake destination: update docs * Normalization for Snowflake destination: added unit tests and change file creating process * Normalization for Snowflake destination: renamed property passphrase to password * Snowflake destination: apply changes from normalization * Snowflake destination: clean code * Snowflake destination: clean up * auto-bump connector version [ci skip] Co-authored-by: Edward Gao <edward.gao@airbyte.io> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Jul 28, 2022 · 6b35b23 · 6b35b23
1 parent 65295b7
commit 6b35b23
Show file tree

Hide file tree

Showing 8 changed files with 158 additions and 8 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -270,7 +270,7 @@
 - name: Snowflake
   destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
   dockerRepository: airbyte/destination-snowflake
-  dockerImageTag: 0.4.33
+  dockerImageTag: 0.4.34
   documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake
   icon: snowflake.svg
   resourceRequirements:

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -4404,7 +4404,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-snowflake:0.4.33"
+- dockerImage: "airbyte/destination-snowflake:0.4.34"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake"
     connectionSpecification:
@@ -4473,8 +4473,8 @@
           description: ""
           type: "object"
           oneOf:
-          - type: "object"
-            title: "OAuth2.0"
+          - title: "OAuth2.0"
+            type: "object"
             order: 0
             required:
             - "access_token"
@@ -4507,11 +4507,37 @@
                 title: "Refresh Token"
                 description: "Enter your application's Refresh Token"
                 airbyte_secret: true
+          - title: "Key Pair Authentication"
+            type: "object"
+            order: 1
+            required:
+            - "private_key"
+            properties:
+              auth_type:
+                type: "string"
+                const: "Key Pair Authentication"
+                enum:
+                - "Key Pair Authentication"
+                default: "Key Pair Authentication"
+                order: 0
+              private_key:
+                type: "string"
+                title: "Private Key"
+                description: "RSA Private key to use for Snowflake connection. See\
+                  \ the <a href=\"https://docs.airbyte.io/integrations/destinations/snowflake\"\
+                  >docs</a> for more information on how to obtain this key."
+                multiline: true
+                airbyte_secret: true
+              private_key_password:
+                type: "string"
+                title: "Passphrase (Optional)"
+                description: "Passphrase for private key"
+                airbyte_secret: true
           - title: "Username and Password"
             type: "object"
             required:
             - "password"
-            order: 1
+            order: 2
             properties:
               password:
                 description: "Enter the password associated with the username."

diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile
@@ -20,5 +20,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1
 
 ENV ENABLE_SENTRY true
 
-LABEL io.airbyte.version=0.4.33
+LABEL io.airbyte.version=0.4.34
 LABEL io.airbyte.name=airbyte/destination-snowflake
diff --git a/...wflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/...wflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java
@@ -13,6 +13,7 @@
 import io.airbyte.db.jdbc.JdbcDatabase;
 import io.airbyte.db.jdbc.JdbcUtils;
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.net.URI;
 import java.net.URLEncoder;
 import java.net.http.HttpClient;
@@ -49,6 +50,9 @@ public class SnowflakeDatabase {
       .version(HttpClient.Version.HTTP_2)
       .connectTimeout(Duration.ofSeconds(10))
       .build();
+  public static final String PRIVATE_KEY_FILE_NAME = "rsa_key.p8";
+  public static final String PRIVATE_KEY_FIELD_NAME = "private_key";
+  public static final String PRIVATE_KEY_PASSWORD = "private_key_password";
 
   public static HikariDataSource createDataSource(final JsonNode config) {
     final HikariDataSource dataSource = new HikariDataSource();
@@ -93,6 +97,15 @@ public static HikariDataSource createDataSource(final JsonNode config) {
       dataSource.setUsername(username);
       dataSource.setPassword(credentials.get(JdbcUtils.PASSWORD_KEY).asText());
 
+    } else if (credentials != null && credentials.has(PRIVATE_KEY_FIELD_NAME)) {
+      LOGGER.debug("Login mode with key pair is used");
+      dataSource.setUsername(username);
+      final String privateKeyValue = credentials.get(PRIVATE_KEY_FIELD_NAME).asText();
+      createPrivateKeyFile(PRIVATE_KEY_FILE_NAME, privateKeyValue);
+      properties.put("private_key_file", PRIVATE_KEY_FILE_NAME);
+      if (credentials.has(PRIVATE_KEY_PASSWORD)) {
+        properties.put("private_key_file_pwd", credentials.get(PRIVATE_KEY_PASSWORD).asText());
+      }
     } else {
       LOGGER.warn(
           "Obsolete User/password login mode is used. Please re-create a connection to use the latest connector's version");
@@ -129,6 +142,14 @@ public static HikariDataSource createDataSource(final JsonNode config) {
     return dataSource;
   }
 
+  private static void createPrivateKeyFile(final String fileName, final String fileValue) {
+    try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) {
+      out.print(fileValue);
+    } catch (IOException e) {
+      throw new RuntimeException("Failed to create file for private key");
+    }
+  }
+
   private static String getAccessTokenUsingRefreshToken(final String hostName,
                                                         final String clientId,
                                                         final String clientSecret,

diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json
@@ -62,8 +62,8 @@
         "type": "object",
         "oneOf": [
           {
-            "type": "object",
             "title": "OAuth2.0",
+            "type": "object",
             "order": 0,
             "required": ["access_token", "refresh_token"],
             "properties": {
@@ -100,11 +100,39 @@
               }
             }
           },
+          {
+            "title": "Key Pair Authentication",
+            "type": "object",
+            "order": 1,
+            "required": ["private_key"],
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "Key Pair Authentication",
+                "enum": ["Key Pair Authentication"],
+                "default": "Key Pair Authentication",
+                "order": 0
+              },
+              "private_key": {
+                "type": "string",
+                "title": "Private Key",
+                "description": "RSA Private key to use for Snowflake connection. See the <a href=\"https://docs.airbyte.io/integrations/destinations/snowflake\">docs</a> for more information on how to obtain this key.",
+                "multiline": true,
+                "airbyte_secret": true
+              },
+              "private_key_password": {
+                "type": "string",
+                "title": "Passphrase (Optional)",
+                "description": "Passphrase for private key",
+                "airbyte_secret": true
+              }
+            }
+          },
           {
             "title": "Username and Password",
             "type": "object",
             "required": ["password"],
-            "order": 1,
+            "order": 2,
             "properties": {
               "password": {
                 "description": "Enter the password associated with the username.",

diff --git a/.../airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java b/.../airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java
@@ -23,6 +23,7 @@
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
 import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import io.airbyte.protocol.models.AirbyteCatalog;
+import io.airbyte.protocol.models.AirbyteConnectionStatus;
 import io.airbyte.protocol.models.AirbyteMessage;
 import io.airbyte.protocol.models.CatalogHelpers;
 import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
@@ -188,6 +189,13 @@ public void testBackwardCompatibilityAfterAddingOauth() {
     assertEquals(Status.SUCCEEDED, runCheckWithCatchedException(deprecatedStyleConfig));
   }
 
+  @Test
+  void testCheckWithKeyPairAuth() throws Exception {
+    final JsonNode credentialsJsonString = Jsons.deserialize(IOs.readFile(Path.of("secrets/config_key_pair.json")));
+    final AirbyteConnectionStatus check = new SnowflakeDestination().check(credentialsJsonString);
+    assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, check.getStatus());
+  }
+
   /**
    * This test is disabled because it is very slow, and should only be run manually for now.
    */

diff --git a/...integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java b/...integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java
@@ -8,7 +8,18 @@
 import com.google.common.base.Preconditions;
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.resources.MoreResources;
+import io.airbyte.integrations.standardtest.destination.DataArgumentsProvider;
+import io.airbyte.protocol.models.AirbyteCatalog;
+import io.airbyte.protocol.models.AirbyteMessage;
+import io.airbyte.protocol.models.AirbyteRecordMessage;
+import io.airbyte.protocol.models.CatalogHelpers;
+import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ArgumentsSource;
 
 public class SnowflakeInternalStagingDestinationAcceptanceTest extends SnowflakeInsertDestinationAcceptanceTest {
 
@@ -19,4 +30,34 @@ public JsonNode getStaticConfig() {
     return internalStagingConfig;
   }
 
+  @ParameterizedTest
+  @ArgumentsSource(DataArgumentsProvider.class)
+  public void testSyncWithNormalizationWithKeyPairAuth(final String messagesFilename, final String catalogFilename) throws Exception {
+    testSyncWithNormalizationWithKeyPairAuth(messagesFilename, catalogFilename, "secrets/config_key_pair.json");
+  }
+
+  @ParameterizedTest
+  @ArgumentsSource(DataArgumentsProvider.class)
+  public void testSyncWithNormalizationWithKeyPairEncrypt(final String messagesFilename, final String catalogFilename) throws Exception {
+    testSyncWithNormalizationWithKeyPairAuth(messagesFilename, catalogFilename, "secrets/config_key_pair_encrypted.json");
+  }
+
+  private void testSyncWithNormalizationWithKeyPairAuth(String messagesFilename, String catalogFilename, String configName) throws Exception {
+    if (!normalizationFromSpec()) {
+      return;
+    }
+
+    final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class);
+    final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog);
+    final List<AirbyteMessage> messages = MoreResources.readResource(messagesFilename).lines()
+        .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList());
+
+    final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of(configName)));
+    runSyncAndVerifyStateOutput(config, messages, configuredCatalog, true);
+
+    final String defaultSchema = getDefaultSchema(config);
+    final List<AirbyteRecordMessage> actualMessages = retrieveNormalizedRecords(catalog, defaultSchema);
+    assertSameMessages(messages, actualMessages, true);
+  }
+
 }
diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md
@@ -183,6 +183,31 @@ Field | Description |
 | [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional) | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3` |
 
 
+### Key pair authentication
+    In order to configure key pair authentication you will need a private/public key pair.
+    If you do not have the key pair yet, you can generate one using openssl command line tool
+    Use this command in order to generate an unencrypted private key file:
+
+       `openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -out rsa_key.p8 -nocrypt`
+
+    Alternatively, use this command to generate an encrypted private key file:
+
+      `openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -v1 PBE-SHA1-RC4-128 -out rsa_key.p8`
+
+    Once you have your private key, you need to generate a matching public key.
+    You can do so with the following command:
+
+      `openssl rsa -in rsa_key.p8 -pubout -out rsa_key.pub`
+
+    Finally, you need to add the public key to your Snowflake user account.
+    You can do so with the following SQL command in Snowflake:
+
+      `alter user <user_name> set rsa_public_key=<public_key_value>;`
+
+    and replace <user_name> with your user name and <public_key_value> with your public key.
+
+
+
 To use AWS S3 as the cloud storage, enter the information for the S3 bucket you created in Step 2:
 
 | Field                          | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
@@ -250,6 +275,7 @@ Now that you have set up the Snowflake destination connector, check out the foll
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                             |
 |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.4.34  | 2022-07-23 | [\#14388](https://github.com/airbytehq/airbyte/pull/14388) | Add support for key pair authentication  |
 | 0.4.33  | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable.                                                                                                               |
 | 0.4.32  | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors                                                                                |
 | 0.4.31  | 2022-07-07 | [\#13729](https://github.com/airbytehq/airbyte/pull/13729) | Improve configuration field description                                                                                                             |