🎉 Source Shopify: implement Oauth2.0 for Airbyte-Cloud (#9591)

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9",
   "name": "Shopify",
   "dockerRepository": "airbyte/source-shopify",
-  "dockerImageTag": "0.1.27",
+  "dockerImageTag": "0.1.28",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify",
   "icon": "shopify.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -641,7 +641,7 @@
 - name: Shopify
   sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
   dockerRepository: airbyte/source-shopify
-  dockerImageTag: 0.1.27
+  dockerImageTag: 0.1.28
   documentationUrl: https://docs.airbyte.io/integrations/sources/shopify
   icon: shopify.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -6684,7 +6684,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-shopify:0.1.27"
+- dockerImage: "airbyte/source-shopify:0.1.28"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/shopify"
     connectionSpecification:
@@ -6694,8 +6694,8 @@
       required:
       - "shop"
       - "start_date"
-      - "auth_method"
-      additionalProperties: false
+      - "credentials"
+      additionalProperties: true
       properties:
         shop:
           type: "string"
@@ -6710,24 +6710,21 @@
           examples:
           - "2021-01-01"
           pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
-        auth_method:
+        credentials:
           title: "Shopify Authorization Method"
           type: "object"
           oneOf:
           - type: "object"
             title: "OAuth2.0"
             required:
-            - "client_id"
-            - "client_secret"
-            - "access_token"
             - "auth_method"
             properties:
               auth_method:
                 type: "string"
-                const: "access_token"
+                const: "oauth2.0"
                 enum:
-                - "access_token"
-                default: "access_token"
+                - "oauth2.0"
+                default: "oauth2.0"
                 order: 0
               client_id:
                 type: "string"
@@ -6747,8 +6744,8 @@
           - title: "API Password"
             type: "object"
             required:
-            - "api_password"
             - "auth_method"
+            - "api_password"
             properties:
               auth_method:
                 type: "string"
@@ -6766,17 +6763,52 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-    authSpecification:
-      auth_type: "oauth2.0"
-      oauth2Specification:
-        rootObject:
-        - "auth_method"
-        - "0"
-        oauthFlowInitParameters:
-        - - "client_id"
-        - - "client_secret"
-        oauthFlowOutputParameters:
-        - - "access_token"
+    advanced_auth:
+      auth_flow_type: "oauth2.0"
+      predicate_key:
+      - "credentials"
+      - "auth_method"
+      predicate_value: "oauth2.0"
+      oauth_config_specification:
+        oauth_user_input_from_connector_config_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            shop:
+              type: "string"
+              path_in_connector_config:
+              - "shop"
+        complete_oauth_output_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            access_token:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "access_token"
+        complete_oauth_server_input_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            client_id:
+              type: "string"
+            client_secret:
+              type: "string"
+        complete_oauth_server_output_specification:
+          type: "object"
+          additionalProperties: false
+          properties:
+            client_id:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "client_id"
+            client_secret:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "client_secret"
 - dockerImage: "airbyte/source-shortio:0.1.2"
   spec:
     documentationUrl: "https://developers.short.io/reference"

airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java

@@ -5,7 +5,6 @@
 package io.airbyte.integrations.destination.bigquery;
 
 import static io.airbyte.integrations.destination.bigquery.helpers.LoggerHelper.getJobErrorMessage;
-import static java.util.Objects.isNull;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -172,9 +171,9 @@ public static String getDatasetId(final JsonNode config) {
       String projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText();
       if (!(projectId.equals(projectIdPart))) {
         throw new IllegalArgumentException(String.format(
-          "Project ID included in Dataset ID must match Project ID field's value: Project ID is `%s`, but you specified `%s` in Dataset ID",
-          projectId,
-          projectIdPart));
+            "Project ID included in Dataset ID must match Project ID field's value: Project ID is `%s`, but you specified `%s` in Dataset ID",
+            projectId,
+            projectIdPart));
       }
     }
     // if colonIndex is -1, then this returns the entire string

airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java

@@ -385,6 +385,7 @@ private boolean isTablePartitioned(final BigQuery bigquery, final Dataset datase
   }
 
   private static class DatasetIdResetter {
+
     private Consumer<JsonNode> consumer;
 
     DatasetIdResetter(Consumer<JsonNode> consumer) {
@@ -394,20 +395,20 @@ private static class DatasetIdResetter {
     public void accept(JsonNode config) {
       consumer.accept(config);
     }
+
   }
 
   private static Stream<Arguments> datasetIdResetterProvider() {
     // parameterized test with two dataset-id patterns: `dataset_id` and `project-id:dataset_id`
     return Stream.of(
-      Arguments.arguments(new DatasetIdResetter(config -> {})),
-      Arguments.arguments(new DatasetIdResetter(
-        config -> {
-          String projectId = ((ObjectNode) config).get(BigQueryConsts.CONFIG_PROJECT_ID).asText();
-          String datasetId = ((ObjectNode) config).get(BigQueryConsts.CONFIG_DATASET_ID).asText();
-          ((ObjectNode) config).put(BigQueryConsts.CONFIG_DATASET_ID,
-            String.format("%s:%s", projectId, datasetId));
-        }
-      ))
-    );
+        Arguments.arguments(new DatasetIdResetter(config -> {})),
+        Arguments.arguments(new DatasetIdResetter(
+            config -> {
+              String projectId = ((ObjectNode) config).get(BigQueryConsts.CONFIG_PROJECT_ID).asText();
+              String datasetId = ((ObjectNode) config).get(BigQueryConsts.CONFIG_DATASET_ID).asText();
+              ((ObjectNode) config).put(BigQueryConsts.CONFIG_DATASET_ID,
+                  String.format("%s:%s", projectId, datasetId));
+            })));
   }
+
 }

airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java

@@ -26,7 +26,7 @@ public void init() {
         .put(BigQueryConsts.CONFIG_CREDS, "test_secret")
         .put(BigQueryConsts.CONFIG_DATASET_LOCATION, "US");
   }
-  
+
   @ParameterizedTest
   @MethodSource("validBigQueryIdProvider")
   public void testGetDatasetIdSuccess(String projectId, String datasetId, String expected) throws Exception {
@@ -66,4 +66,5 @@ private static Stream<Arguments> invalidBigQueryIdProvider() {
         Arguments.arguments("my-project", "your-project:my_dataset",
             "Project ID included in Dataset ID must match Project ID field's value: Project ID is `my-project`, but you specified `your-project` in Dataset ID"));
   }
+
 }

airbyte-integrations/connectors/source-shopify/Dockerfile

@@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.27
+LABEL io.airbyte.version=0.1.28
 LABEL io.airbyte.name=airbyte/source-shopify

airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json

@@ -1,7 +1,7 @@
 {
   "shop": "SHOP_NAME",
   "start_date": "2020-11-01",
-  "auth_method": {
+  "credentials": {
     "auth_method": "api_password",
     "api_password": "SOME_API_PASSWORD"
   }

airbyte-integrations/connectors/source-shopify/integration_tests/invalid_oauth_config.json

@@ -1,8 +1,8 @@
 {
   "shop": "SHOP_NAME",
   "start_date": "2020-11-01",
-  "auth_method": {
-    "auth_method": "access_token",
+  "credentials": {
+    "auth_method": "oauth2.0",
     "client_id": "SOME_CLIENT_ID",
     "client_secret": "SOME_CLIENT_SECRET",
     "access_token": "SOME_ACCESS_TOKEN"

airbyte-integrations/connectors/source-shopify/source_shopify/auth.py

@@ -30,12 +30,12 @@ def __init__(self, config: Mapping[str, Any]):
     def get_auth_header(self) -> Mapping[str, Any]:
 
         auth_header: str = "X-Shopify-Access-Token"
-        auth_method: Dict = self.config["auth_method"]
-        auth_option: str = auth_method.get("auth_method")
+        credentials: Dict = self.config["credentials"]
+        auth_method: str = credentials.get("auth_method")
 
-        if auth_option == "access_token":
-            return {auth_header: auth_method.get("access_token")}
-        elif auth_option == "api_password":
-            return {auth_header: auth_method.get("api_password")}
+        if auth_method == "oauth2.0":
+            return {auth_header: credentials.get("access_token")}
+        elif auth_method == "api_password":
+            return {auth_header: credentials.get("api_password")}
         else:
-            raise NotImplementedAuth(auth_option)
+            raise NotImplementedAuth(auth_method)

airbyte-integrations/connectors/source-shopify/source_shopify/spec.json

@@ -4,8 +4,8 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Shopify Source CDK Specifications",
     "type": "object",
-    "required": ["shop", "start_date", "auth_method"],
-    "additionalProperties": false,
+    "required": ["shop", "start_date", "credentials"],
+    "additionalProperties": true,
     "properties": {
       "shop": {
         "type": "string",
@@ -19,25 +19,20 @@
         "examples": ["2021-01-01"],
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
       },
-      "auth_method": {
+      "credentials": {
         "title": "Shopify Authorization Method",
         "type": "object",
         "oneOf": [
           {
             "type": "object",
             "title": "OAuth2.0",
-            "required": [
-              "client_id",
-              "client_secret",
-              "access_token",
-              "auth_method"
-            ],
+            "required": ["auth_method"],
             "properties": {
               "auth_method": {
                 "type": "string",
-                "const": "access_token",
-                "enum": ["access_token"],
-                "default": "access_token",
+                "const": "oauth2.0",
+                "enum": ["oauth2.0"],
+                "default": "oauth2.0",
                 "order": 0
               },
               "client_id": {
@@ -63,7 +58,7 @@
           {
             "title": "API Password",
             "type": "object",
-            "required": ["api_password", "auth_method"],
+            "required": ["auth_method", "api_password"],
             "properties": {
               "auth_method": {
                 "type": "string",
@@ -84,12 +79,57 @@
       }
     }
   },
-  "authSpecification": {
-    "auth_type": "oauth2.0",
-    "oauth2Specification": {
-      "rootObject": ["auth_method", 0],
-      "oauthFlowInitParameters": [["client_id"], ["client_secret"]],
-      "oauthFlowOutputParameters": [["access_token"]]
+  "advanced_auth": {
+    "auth_flow_type": "oauth2.0",
+    "predicate_key": ["credentials", "auth_method"],
+    "predicate_value": "oauth2.0",
+    "oauth_config_specification": {
+      "complete_oauth_output_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "access_token": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "access_token"]
+          }
+        }
+      },
+      "complete_oauth_server_input_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "client_id": {
+            "type": "string"
+          },
+          "client_secret": {
+            "type": "string"
+          }
+        }
+      },
+      "complete_oauth_server_output_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "client_id": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_id"]
+          },
+          "client_secret": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_secret"]
+          }
+        }
+      },
+      "oauth_user_input_from_connector_config_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "shop": {
+            "type": "string",
+            "path_in_connector_config": ["shop"]
+          }
+        }
+      }
     }
   }
 }