From c308c2f7cd830bb696c9d96352c82cf280db35f1 Mon Sep 17 00:00:00 2001 From: Oleksandr Sheheda Date: Wed, 13 Jul 2022 22:29:21 +0300 Subject: [PATCH] Source TikTok Marketing : Access token is null #14299 (#14461) * Replaced List with Set in order to avoid credentials loosing for case when we have multiple options with the same key on spec * returned back List to save the order of fields, added distinct() to have a unique path values * clean up * [14299] Source TikTok Marketing : Access token is null added unit tests * [14299] Source TikTok Marketing : Access token is null updated javadoc * Format Co-authored-by: Tuhai Maksym Co-authored-by: Benoit Moriceau --- .../split_secrets/SecretsHelpers.java | 5 +- .../JsonSecretsProcessorTest.java | 3 + .../split_secrets/SecretsHelpersTest.java | 2 + .../test_cases/OneOfSecretTestCase.java | 54 ++++++++++++++ .../test/resources/oneof_secret/expected.json | 9 +++ .../test/resources/oneof_secret/expectedPaths | 1 + .../resources/oneof_secret/full_config.json | 9 +++ .../oneof_secret/partial_config.json | 17 +++++ .../src/test/resources/oneof_secret/spec.json | 74 +++++++++++++++++++ .../resources/oneof_secret/update_config.json | 6 ++ .../oneof_secret/updated_partial_config.json | 8 ++ 11 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/test_cases/OneOfSecretTestCase.java create mode 100644 airbyte-config/config-persistence/src/test/resources/oneof_secret/expected.json create mode 100644 airbyte-config/config-persistence/src/test/resources/oneof_secret/expectedPaths create mode 100644 airbyte-config/config-persistence/src/test/resources/oneof_secret/full_config.json create mode 100644 airbyte-config/config-persistence/src/test/resources/oneof_secret/partial_config.json create mode 100644 airbyte-config/config-persistence/src/test/resources/oneof_secret/spec.json create mode 100644 airbyte-config/config-persistence/src/test/resources/oneof_secret/update_config.json create mode 100644 airbyte-config/config-persistence/src/test/resources/oneof_secret/updated_partial_config.json diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java index eef92dcdfd97..4e5debdbb0b1 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java @@ -167,8 +167,8 @@ public static SplitSecretConfig splitAndUpdateConfig(final Supplier uuidSu } /** - * This returns all the path to the airbyte secrets based on a schema spec. The path will be return - * in an ascending alphabetical order. + * This returns all the unique path to the airbyte secrets based on a schema spec. The path will be + * return in an ascending alphabetical order. */ public static List getSortedSecretPaths(final JsonNode spec) { return JsonSchemas.collectPathsThatMeetCondition( @@ -178,6 +178,7 @@ public static List getSortedSecretPaths(final JsonNode spec) { .anyMatch(field -> field.getKey().equals(JsonSecretsProcessor.AIRBYTE_SECRET_FIELD))) .stream() .map(JsonPaths::mapJsonSchemaPathToJsonPath) + .distinct() .sorted() .toList(); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessorTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessorTest.java index 52ce8030669c..8302226800a1 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessorTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessorTest.java @@ -186,6 +186,7 @@ class JsonSecretsProcessorTest { private static final String ARRAY_OF_ONEOF = "array_of_oneof"; private static final String NESTED_OBJECT = "nested_object"; private static final String NESTED_ONEOF = "nested_oneof"; + private static final String ONE_OF_SECRET = "oneof_secret"; private static final String ONE_OF = "oneof"; private static final String OPTIONAL_PASSWORD = "optional_password"; private static final String POSTGRES_SSH_KEY = "postgres_ssh_key"; @@ -334,6 +335,8 @@ private static Stream scenarioProvider() { Arguments.of(NESTED_ONEOF, false), Arguments.of(ONE_OF, true), Arguments.of(ONE_OF, false), + Arguments.of(ONE_OF_SECRET, true), + Arguments.of(ONE_OF_SECRET, false), Arguments.of(OPTIONAL_PASSWORD, true), Arguments.of(OPTIONAL_PASSWORD, false), Arguments.of(POSTGRES_SSH_KEY, true), diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/SecretsHelpersTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/SecretsHelpersTest.java index 59b314ddd6b7..4658730bd365 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/SecretsHelpersTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/SecretsHelpersTest.java @@ -14,6 +14,7 @@ import io.airbyte.config.persistence.split_secrets.test_cases.ArrayTestCase; import io.airbyte.config.persistence.split_secrets.test_cases.NestedObjectTestCase; import io.airbyte.config.persistence.split_secrets.test_cases.NestedOneOfTestCase; +import io.airbyte.config.persistence.split_secrets.test_cases.OneOfSecretTestCase; import io.airbyte.config.persistence.split_secrets.test_cases.OneOfTestCase; import io.airbyte.config.persistence.split_secrets.test_cases.OptionalPasswordTestCase; import io.airbyte.config.persistence.split_secrets.test_cases.PostgresSshKeyTestCase; @@ -61,6 +62,7 @@ private static Stream provideTestCases() { new SimpleTestCase(), new NestedObjectTestCase(), new OneOfTestCase(), + new OneOfSecretTestCase(), new ArrayTestCase(), new ArrayOneOfTestCase(), new NestedOneOfTestCase(), diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/test_cases/OneOfSecretTestCase.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/test_cases/OneOfSecretTestCase.java new file mode 100644 index 000000000000..65fa3c0cdd0c --- /dev/null +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/split_secrets/test_cases/OneOfSecretTestCase.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence.split_secrets.test_cases; + +import io.airbyte.config.persistence.split_secrets.SecretCoordinate; +import io.airbyte.config.persistence.split_secrets.SecretPersistence; +import io.airbyte.config.persistence.split_secrets.SecretsHelpersTest; +import io.airbyte.config.persistence.split_secrets.SecretsTestCase; +import java.util.Map; +import java.util.function.Consumer; + +public class OneOfSecretTestCase implements SecretsTestCase { + + @Override + public String getName() { + return "oneof_secret"; + } + + @Override + public Map getFirstSecretMap() { + return Map.of( + new SecretCoordinate(PREFIX + SecretsHelpersTest.WORKSPACE_ID + SECRET + SecretsHelpersTest.UUIDS.get(0), 1), "access_token_1", + new SecretCoordinate(PREFIX + SecretsHelpersTest.WORKSPACE_ID + SECRET + SecretsHelpersTest.UUIDS.get(1), 1), "clientId_1", + new SecretCoordinate(PREFIX + SecretsHelpersTest.WORKSPACE_ID + SECRET + SecretsHelpersTest.UUIDS.get(2), 1), "client_secret_1", + new SecretCoordinate(PREFIX + SecretsHelpersTest.WORKSPACE_ID + SECRET + SecretsHelpersTest.UUIDS.get(3), 1), "refresh_token_1"); + } + + @Override + public Map getSecondSecretMap() { + return Map.of( + new SecretCoordinate(PREFIX + SecretsHelpersTest.WORKSPACE_ID + SECRET + SecretsHelpersTest.UUIDS.get(0), 2), "access_token_2"); + } + + @Override + public Consumer getPersistenceUpdater() { + return secretPersistence -> { + secretPersistence.write( + new SecretCoordinate(PREFIX + SecretsHelpersTest.WORKSPACE_ID + SECRET + SecretsHelpersTest.UUIDS.get(0), 1), + "access_token_1"); + secretPersistence.write( + new SecretCoordinate(PREFIX + SecretsHelpersTest.WORKSPACE_ID + SECRET + SecretsHelpersTest.UUIDS.get(1), 1), + "clientId_1"); + secretPersistence.write( + new SecretCoordinate(PREFIX + SecretsHelpersTest.WORKSPACE_ID + SECRET + SecretsHelpersTest.UUIDS.get(2), 1), + "client_secret_1"); + secretPersistence.write( + new SecretCoordinate(PREFIX + SecretsHelpersTest.WORKSPACE_ID + SECRET + SecretsHelpersTest.UUIDS.get(3), 1), + "refresh_token_1"); + }; + } + +} diff --git a/airbyte-config/config-persistence/src/test/resources/oneof_secret/expected.json b/airbyte-config/config-persistence/src/test/resources/oneof_secret/expected.json new file mode 100644 index 000000000000..54f692e42a9e --- /dev/null +++ b/airbyte-config/config-persistence/src/test/resources/oneof_secret/expected.json @@ -0,0 +1,9 @@ +{ + "credentials": { + "auth_method": "oauth2.0", + "client_id": "**********", + "client_secret": "**********", + "access_token": "**********", + "refresh_token": "**********" + } +} diff --git a/airbyte-config/config-persistence/src/test/resources/oneof_secret/expectedPaths b/airbyte-config/config-persistence/src/test/resources/oneof_secret/expectedPaths new file mode 100644 index 000000000000..2e8f07aa898a --- /dev/null +++ b/airbyte-config/config-persistence/src/test/resources/oneof_secret/expectedPaths @@ -0,0 +1 @@ +$.credentials.access_token;$.credentials.client_id;$.credentials.client_secret;$.credentials.refresh_token; diff --git a/airbyte-config/config-persistence/src/test/resources/oneof_secret/full_config.json b/airbyte-config/config-persistence/src/test/resources/oneof_secret/full_config.json new file mode 100644 index 000000000000..a8793dc672a5 --- /dev/null +++ b/airbyte-config/config-persistence/src/test/resources/oneof_secret/full_config.json @@ -0,0 +1,9 @@ +{ + "credentials": { + "auth_method": "oauth2.0", + "client_id": "clientId_1", + "client_secret": "client_secret_1", + "access_token": "access_token_1", + "refresh_token": "refresh_token_1" + } +} diff --git a/airbyte-config/config-persistence/src/test/resources/oneof_secret/partial_config.json b/airbyte-config/config-persistence/src/test/resources/oneof_secret/partial_config.json new file mode 100644 index 000000000000..0eeaac9caca7 --- /dev/null +++ b/airbyte-config/config-persistence/src/test/resources/oneof_secret/partial_config.json @@ -0,0 +1,17 @@ +{ + "credentials": { + "auth_method": "oauth2.0", + "client_id": { + "_secret": "airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_2c2ef2b3-259a-4e73-96d1-f56dacee2e5e_v1" + }, + "client_secret": { + "_secret": "airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_1206db5b-b968-4df1-9a76-f3fcdae7e307_v1" + }, + "access_token": { + "_secret": "airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_9eba44d8-51e7-48f1-bde2-619af0e42c22_v1" + }, + "refresh_token": { + "_secret": "airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_c03ef566-79a7-4e77-b6f3-d23d2528f25a_v1" + } + } +} diff --git a/airbyte-config/config-persistence/src/test/resources/oneof_secret/spec.json b/airbyte-config/config-persistence/src/test/resources/oneof_secret/spec.json new file mode 100644 index 000000000000..d8a54edba8e0 --- /dev/null +++ b/airbyte-config/config-persistence/src/test/resources/oneof_secret/spec.json @@ -0,0 +1,74 @@ +{ + "type": "object", + "properties": { + "credentials": { + "title": "Authorization Method", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": [ + "client_id", + "client_secret", + "access_token", + "refresh_token" + ], + "properties": { + "auth_method": { + "type": "string", + "const": "oauth2.0", + "enum": ["oauth2.0"], + "default": "oauth2.0", + "order": 0 + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The Client ID of your application.", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "The Client Secret of your application.", + "airbyte_secret": true + }, + "access_token": { + "type": "string", + "title": "Access Token", + "description": "Access Token for making authenticated requests.", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "title": "Refresh Token", + "description": "Refresh Token to renew the expired Access Token.", + "default": "", + "airbyte_secret": true + } + } + }, + { + "title": "Access Token", + "type": "object", + "required": ["access_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "access_token", + "enum": ["access_token"], + "default": "access_token", + "order": 0 + }, + "access_token": { + "type": "string", + "title": "Access Token", + "airbyte_secret": true + } + } + } + ] + } + } +} diff --git a/airbyte-config/config-persistence/src/test/resources/oneof_secret/update_config.json b/airbyte-config/config-persistence/src/test/resources/oneof_secret/update_config.json new file mode 100644 index 000000000000..c6df01a70110 --- /dev/null +++ b/airbyte-config/config-persistence/src/test/resources/oneof_secret/update_config.json @@ -0,0 +1,6 @@ +{ + "credentials": { + "auth_method": "access_token", + "access_token": "access_token_2" + } +} diff --git a/airbyte-config/config-persistence/src/test/resources/oneof_secret/updated_partial_config.json b/airbyte-config/config-persistence/src/test/resources/oneof_secret/updated_partial_config.json new file mode 100644 index 000000000000..1064cac031e5 --- /dev/null +++ b/airbyte-config/config-persistence/src/test/resources/oneof_secret/updated_partial_config.json @@ -0,0 +1,8 @@ +{ + "credentials": { + "auth_method": "access_token", + "access_token": { + "_secret": "airbyte_workspace_e0eb0554-ffe0-4e9c-9dc0-ed7f52023eb2_secret_9eba44d8-51e7-48f1-bde2-619af0e42c22_v2" + } + } +}