diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index e6f4974cc21a..b5e614299653 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -270,7 +270,7 @@ - name: Snowflake destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.4.33 + dockerImageTag: 0.4.34 documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake icon: snowflake.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index d94db5b906f2..cb8af723a6f9 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -4404,7 +4404,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-snowflake:0.4.33" +- dockerImage: "airbyte/destination-snowflake:0.4.34" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake" connectionSpecification: @@ -4473,8 +4473,8 @@ description: "" type: "object" oneOf: - - type: "object" - title: "OAuth2.0" + - title: "OAuth2.0" + type: "object" order: 0 required: - "access_token" @@ -4507,11 +4507,37 @@ title: "Refresh Token" description: "Enter your application's Refresh Token" airbyte_secret: true + - title: "Key Pair Authentication" + type: "object" + order: 1 + required: + - "private_key" + properties: + auth_type: + type: "string" + const: "Key Pair Authentication" + enum: + - "Key Pair Authentication" + default: "Key Pair Authentication" + order: 0 + private_key: + type: "string" + title: "Private Key" + description: "RSA Private key to use for Snowflake connection. See\ + \ the docs for more information on how to obtain this key." + multiline: true + airbyte_secret: true + private_key_password: + type: "string" + title: "Passphrase (Optional)" + description: "Passphrase for private key" + airbyte_secret: true - title: "Username and Password" type: "object" required: - "password" - order: 1 + order: 2 properties: password: description: "Enter the password associated with the username." diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 56c6f6d0b97f..8f7dfc05fdff 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -20,5 +20,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=0.4.33 +LABEL io.airbyte.version=0.4.34 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java index 533517dbb6ab..02742c8ef53c 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java @@ -13,6 +13,7 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; import java.io.IOException; +import java.io.PrintWriter; import java.net.URI; import java.net.URLEncoder; import java.net.http.HttpClient; @@ -49,6 +50,9 @@ public class SnowflakeDatabase { .version(HttpClient.Version.HTTP_2) .connectTimeout(Duration.ofSeconds(10)) .build(); + public static final String PRIVATE_KEY_FILE_NAME = "rsa_key.p8"; + public static final String PRIVATE_KEY_FIELD_NAME = "private_key"; + public static final String PRIVATE_KEY_PASSWORD = "private_key_password"; public static HikariDataSource createDataSource(final JsonNode config) { final HikariDataSource dataSource = new HikariDataSource(); @@ -93,6 +97,15 @@ public static HikariDataSource createDataSource(final JsonNode config) { dataSource.setUsername(username); dataSource.setPassword(credentials.get(JdbcUtils.PASSWORD_KEY).asText()); + } else if (credentials != null && credentials.has(PRIVATE_KEY_FIELD_NAME)) { + LOGGER.debug("Login mode with key pair is used"); + dataSource.setUsername(username); + final String privateKeyValue = credentials.get(PRIVATE_KEY_FIELD_NAME).asText(); + createPrivateKeyFile(PRIVATE_KEY_FILE_NAME, privateKeyValue); + properties.put("private_key_file", PRIVATE_KEY_FILE_NAME); + if (credentials.has(PRIVATE_KEY_PASSWORD)) { + properties.put("private_key_file_pwd", credentials.get(PRIVATE_KEY_PASSWORD).asText()); + } } else { LOGGER.warn( "Obsolete User/password login mode is used. Please re-create a connection to use the latest connector's version"); @@ -129,6 +142,14 @@ public static HikariDataSource createDataSource(final JsonNode config) { return dataSource; } + private static void createPrivateKeyFile(final String fileName, final String fileValue) { + try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { + out.print(fileValue); + } catch (IOException e) { + throw new RuntimeException("Failed to create file for private key"); + } + } + private static String getAccessTokenUsingRefreshToken(final String hostName, final String clientId, final String clientSecret, diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index 1c5be9376d66..e8345277de5b 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -62,8 +62,8 @@ "type": "object", "oneOf": [ { - "type": "object", "title": "OAuth2.0", + "type": "object", "order": 0, "required": ["access_token", "refresh_token"], "properties": { @@ -100,11 +100,39 @@ } } }, + { + "title": "Key Pair Authentication", + "type": "object", + "order": 1, + "required": ["private_key"], + "properties": { + "auth_type": { + "type": "string", + "const": "Key Pair Authentication", + "enum": ["Key Pair Authentication"], + "default": "Key Pair Authentication", + "order": 0 + }, + "private_key": { + "type": "string", + "title": "Private Key", + "description": "RSA Private key to use for Snowflake connection. See the docs for more information on how to obtain this key.", + "multiline": true, + "airbyte_secret": true + }, + "private_key_password": { + "type": "string", + "title": "Passphrase (Optional)", + "description": "Passphrase for private key", + "airbyte_secret": true + } + } + }, { "title": "Username and Password", "type": "object", "required": ["password"], - "order": 1, + "order": 2, "properties": { "password": { "description": "Enter the password associated with the username.", diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java index aa5cfb450be1..44c675f2c31d 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java @@ -23,6 +23,7 @@ import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; @@ -188,6 +189,13 @@ public void testBackwardCompatibilityAfterAddingOauth() { assertEquals(Status.SUCCEEDED, runCheckWithCatchedException(deprecatedStyleConfig)); } + @Test + void testCheckWithKeyPairAuth() throws Exception { + final JsonNode credentialsJsonString = Jsons.deserialize(IOs.readFile(Path.of("secrets/config_key_pair.json"))); + final AirbyteConnectionStatus check = new SnowflakeDestination().check(credentialsJsonString); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, check.getStatus()); + } + /** * This test is disabled because it is very slow, and should only be run manually for now. */ diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java index fa33f5c97cc5..609882a6a59d 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java @@ -8,7 +8,18 @@ import com.google.common.base.Preconditions; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.standardtest.destination.DataArgumentsProvider; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ArgumentsSource; public class SnowflakeInternalStagingDestinationAcceptanceTest extends SnowflakeInsertDestinationAcceptanceTest { @@ -19,4 +30,34 @@ public JsonNode getStaticConfig() { return internalStagingConfig; } + @ParameterizedTest + @ArgumentsSource(DataArgumentsProvider.class) + public void testSyncWithNormalizationWithKeyPairAuth(final String messagesFilename, final String catalogFilename) throws Exception { + testSyncWithNormalizationWithKeyPairAuth(messagesFilename, catalogFilename, "secrets/config_key_pair.json"); + } + + @ParameterizedTest + @ArgumentsSource(DataArgumentsProvider.class) + public void testSyncWithNormalizationWithKeyPairEncrypt(final String messagesFilename, final String catalogFilename) throws Exception { + testSyncWithNormalizationWithKeyPairAuth(messagesFilename, catalogFilename, "secrets/config_key_pair_encrypted.json"); + } + + private void testSyncWithNormalizationWithKeyPairAuth(String messagesFilename, String catalogFilename, String configName) throws Exception { + if (!normalizationFromSpec()) { + return; + } + + final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); + final List messages = MoreResources.readResource(messagesFilename).lines() + .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); + + final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of(configName))); + runSyncAndVerifyStateOutput(config, messages, configuredCatalog, true); + + final String defaultSchema = getDefaultSchema(config); + final List actualMessages = retrieveNormalizedRecords(catalog, defaultSchema); + assertSameMessages(messages, actualMessages, true); + } + } diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index fca173f780f7..fd96bcda652c 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -183,6 +183,31 @@ Field | Description | | [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional) | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3` | +### Key pair authentication + In order to configure key pair authentication you will need a private/public key pair. + If you do not have the key pair yet, you can generate one using openssl command line tool + Use this command in order to generate an unencrypted private key file: + + `openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -out rsa_key.p8 -nocrypt` + + Alternatively, use this command to generate an encrypted private key file: + + `openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -v1 PBE-SHA1-RC4-128 -out rsa_key.p8` + + Once you have your private key, you need to generate a matching public key. + You can do so with the following command: + + `openssl rsa -in rsa_key.p8 -pubout -out rsa_key.pub` + + Finally, you need to add the public key to your Snowflake user account. + You can do so with the following SQL command in Snowflake: + + `alter user set rsa_public_key=;` + + and replace with your user name and with your public key. + + + To use AWS S3 as the cloud storage, enter the information for the S3 bucket you created in Step 2: | Field | Description | @@ -250,6 +275,7 @@ Now that you have set up the Snowflake destination connector, check out the foll | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.4.34 | 2022-07-23 | [\#14388](https://github.com/airbytehq/airbyte/pull/14388) | Add support for key pair authentication | | 0.4.33 | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable. | | 0.4.32 | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors | | 0.4.31 | 2022-07-07 | [\#13729](https://github.com/airbytehq/airbyte/pull/13729) | Improve configuration field description |