From 0cda5fa7c5ab9e30f8fc0daee8c25044426ba0b3 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Mon, 8 Aug 2022 10:06:37 +0300 Subject: [PATCH] updated normalization version for postgres destination --- .../bases/base-normalization/Dockerfile | 2 +- .../transform_config/transform.py | 24 ++++++++++++++++--- .../NormalizationRunnerFactory.java | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index eb2efbdcc31c..23656c080c42 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.2.16 +LABEL io.airbyte.version=0.2.17 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index 3d5543039e1b..a3c1a77bbc4d 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -8,6 +8,7 @@ import os import pkgutil import socket +import subprocess from typing import Any, Dict import yaml @@ -64,6 +65,13 @@ def transform(self, integration_type: DestinationType, config: Dict[str, Any]): return base_profile + @staticmethod + def create_file(name, content): + f = open(name, "x") + f.write(content) + f.close() + return os.path.abspath(f.name) + @staticmethod def is_ssh_tunnelling(config: Dict[str, Any]) -> bool: tunnel_methods = ["SSH_KEY_AUTH", "SSH_PASSWORD_AUTH"] @@ -167,9 +175,19 @@ def transform_postgres(config: Dict[str, Any]): "threads": 8, } - # if unset, we assume true. - if config.get("ssl", True): - config["sslmode"] = "require" + ssl = config.get("ssl") + if ssl: + ssl_mode = config.get("ssl_mode", "allow") + dbt_config["sslmode"] = ssl_mode.get("mode") + if ssl_mode["mode"] == "verify-ca": + TransformConfig.create_file("ca.crt", ssl_mode["ca_certificate"]) + dbt_config["sslrootcert"] = "ca.crt" + elif ssl_mode["mode"] == "verify-full": + dbt_config["sslrootcert"] = TransformConfig.create_file("ca.crt", ssl_mode["ca_certificate"]) + dbt_config["sslcert"] = TransformConfig.create_file("client.crt", ssl_mode["client_certificate"]) + client_key = TransformConfig.create_file("client.key", ssl_mode["client_key"]) + subprocess.call("openssl pkcs8 -topk8 -inform PEM -in client.key -outform DER -out client.pk8 -nocrypt", shell=True) + dbt_config["sslkey"] = client_key.replace("client.key", "client.pk8") return dbt_config diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index 69012e409a72..fa86ddf27a9d 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.2.16"; + public static final String NORMALIZATION_VERSION = "0.2.17"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder()