Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Normalization for Snowflake destination: added support for key pair authentication #14792

Merged
2 changes: 1 addition & 1 deletion airbyte-integrations/bases/base-normalization/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,5 +28,5 @@ WORKDIR /airbyte
ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh"
ENTRYPOINT ["/airbyte/entrypoint.sh"]

LABEL io.airbyte.version=0.2.9
LABEL io.airbyte.version=0.2.10
LABEL io.airbyte.name=airbyte/normalization
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,12 @@ def transform_snowflake(config: Dict[str, Any]):
dbt_config["oauth_client_id"] = credentials["client_id"]
dbt_config["oauth_client_secret"] = credentials["client_secret"]
dbt_config["token"] = credentials["refresh_token"]
elif credentials.get("private_key"):
with open("private_key_path.txt", "w") as f:
f.write(credentials["private_key"])
dbt_config["private_key_path"] = "private_key_path.txt"
if credentials.get("password"):
dbt_config["private_key_passphrase"] = credentials["password"]
elif credentials.get("password"):
dbt_config["password"] = credentials["password"]
else:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,44 @@ def test_transform_snowflake_oauth(self):
assert expected == actual
assert extract_schema(actual) == "AIRBYTE_SCHEMA"

def test_transform_snowflake_key_pair(self):

input = {
"host": "http://123abc.us-east-7.aws.snowflakecomputing.com",
"role": "AIRBYTE_ROLE",
"warehouse": "AIRBYTE_WAREHOUSE",
"database": "AIRBYTE_DATABASE",
"schema": "AIRBYTE_SCHEMA",
"username": "AIRBYTE_USER",
"credentials": {
"private_key": "AIRBYTE_PRIVATE_KEY",
"passphrase": "AIRBYTE_PRIVATE_KEY_PASSWORD",
},
}

actual = TransformConfig().transform_snowflake(input)
expected = {
"account": "123abc.us-east-7.aws",
"client_session_keep_alive": False,
"database": "AIRBYTE_DATABASE",
"query_tag": "normalization",
"role": "AIRBYTE_ROLE",
"schema": "AIRBYTE_SCHEMA",
"threads": 5,
"retry_all": True,
"retry_on_database_errors": True,
"connect_retries": 3,
"connect_timeout": 15,
"type": "snowflake",
"user": "AIRBYTE_USER",
"warehouse": "AIRBYTE_WAREHOUSE",
"private_key_path": "private_key_path.txt",
"private_key_passphrase": "AIRBYTE_PRIVATE_KEY_PASSWORD",
}

assert expected == actual
assert extract_schema(actual) == "AIRBYTE_SCHEMA"

def test_transform_mysql(self):
input = {
"type": "mysql5",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
public class NormalizationRunnerFactory {

public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization";
public static final String NORMALIZATION_VERSION = "0.2.9";
public static final String NORMALIZATION_VERSION = "0.2.10";

static final Map<String, ImmutablePair<String, DefaultNormalizationRunner.DestinationType>> NORMALIZATION_MAPPING =
ImmutableMap.<String, ImmutablePair<String, DefaultNormalizationRunner.DestinationType>>builder()
Expand Down
3 changes: 2 additions & 1 deletion docs/understanding-airbyte/basic-normalization.md
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,8 @@ Therefore, in order to "upgrade" to the desired normalization version, you need

| Airbyte Version | Normalization Version | Date | Pull Request | Subject |
|:----------------| :--- | :--- | :--- | :--- |
| | 0.2.9 | 2022-07-06 | [\#14485](https://github.com/airbytehq/airbyte/pull/14485/) | BigQuery partition pruning otimization |
| | 0.2.10 | 2022-07-18 | [\#14792](https://github.com/airbytehq/airbyte/pull/14792) | Add support for key pair auth for snowflake |
| | 0.2.9 | 2022-07-06 | [\#14485](https://github.com/airbytehq/airbyte/pull/14485) | BigQuery partition pruning otimization |
| | 0.2.8 | 2022-07-13 | [\#14522](https://github.com/airbytehq/airbyte/pull/14522) | BigQuery replaces `NULL` array entries with the string value `"NULL"` |
| | 0.2.7 | 2022-07-05 | [\#11694](https://github.com/airbytehq/airbyte/pull/11694) | Do not return NULL for MySQL column values > 512 chars |
| | 0.2.6 | 2022-06-16 | [\#13894](https://github.com/airbytehq/airbyte/pull/13894) | Fix incorrect jinja2 macro `json_extract_array` call |
Expand Down