🐛 🎉 Source Airtable: migrate to the Metadata API for dynamic schema…

… generation (#20846)
airbytehq · Jan 9, 2023 · 98ee1c2 · 98ee1c2
1 parent 86157be
commit 98ee1c2
Show file tree

Hide file tree

Showing 18 changed files with 695 additions and 286 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -16,7 +16,7 @@
 - name: Airtable
   sourceDefinitionId: 14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212
   dockerRepository: airbyte/source-airtable
-  dockerImageTag: 0.1.3
+  dockerImageTag: 1.0.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/airtable
   icon: airtable.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -178,7 +178,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-airtable:0.1.3"
+- dockerImage: "airbyte/source-airtable:1.0.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/airtable"
     connectionSpecification:
@@ -187,35 +187,16 @@
       type: "object"
       required:
       - "api_key"
-      - "base_id"
-      - "tables"
       properties:
         api_key:
           type: "string"
-          description: "The API Key for the Airtable account. See the <a href=\"https://support.airtable.com/hc/en-us/articles/219046777-How-do-I-get-my-API-key-\"\
-            >Support Guide</a> for more information on how to obtain this key."
+          description: "The API Key or PAT for the Airtable account. See the <a href=\"\
+            https://airtable.com/developers/web/guides/personal-access-tokens\">Support\
+            \ Guide</a> for more information on how to obtain this key."
           title: "API Key"
           airbyte_secret: true
           examples:
           - "key1234567890"
-        base_id:
-          type: "string"
-          description: "The Base ID to integrate the data from. You can find the Base\
-            \ ID following the link <a href=\"https://airtable.com/api\">Airtable\
-            \ API</a>, log in to your account, select the base you need and find Base\
-            \ ID in the docs."
-          title: "Base ID"
-          examples:
-          - "app1234567890"
-        tables:
-          type: "array"
-          items:
-            type: "string"
-          description: "The list of Tables to integrate."
-          title: "Tables"
-          examples:
-          - "table 1"
-          - "table 2"
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []

diff --git a/airbyte-integrations/connectors/source-airtable/Dockerfile b/airbyte-integrations/connectors/source-airtable/Dockerfile
@@ -34,5 +34,5 @@ COPY source_airtable ./source_airtable
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.3
+LABEL io.airbyte.version=1.0.0
 LABEL io.airbyte.name=airbyte/source-airtable
diff --git a/airbyte-integrations/connectors/source-airtable/README.md b/airbyte-integrations/connectors/source-airtable/README.md
@@ -103,7 +103,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce
 If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
 To run your integration tests with acceptance tests, from the connector root, run
 ```
-python -m pytest integration_tests -p integration_tests.acceptance
+docker build . --no-cache -t airbyte/source-airtable:dev \
+&& python -m pytest integration_tests -p integration_tests.acceptance
 ```
 To run your integration tests with docker
 

diff --git a/airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml b/airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml
@@ -1,20 +1,28 @@
 # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
 # for more information about how to configure these tests
 connector_image: airbyte/source-airtable:dev
-tests:
+acceptance_tests:
   spec:
-    - spec_path: "source_airtable/spec.json"
+    tests:
+      - spec_path: "source_airtable/spec.json"
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
+    tests:
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
   discovery:
-    - config_path: "secrets/config.json"
+    tests:
+      - config_path: "secrets/config.json"
+      # bypassed this check, because discovery mechanism was changed
+        backward_compatibility_tests_config:
+          disable_for_version: "0.1.3"
   basic_read:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      empty_streams: []
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        empty_streams: []
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-airtable/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-airtable/integration_tests/configured_catalog.json
@@ -2,7 +2,63 @@
   "streams": [
     {
       "stream": {
-        "name": "Table 1",
+        "name": "users/table_1",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": "object",
+          "properties": {}
+        },
+        "supported_sync_modes": ["full_refresh"],
+        "supported_destination_sync_modes": ["overwrite", "append_dedup"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    }, 
+    {
+      "stream": {
+        "name": "users/table_2",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": "object",
+          "properties": {}
+        },
+        "supported_sync_modes": ["full_refresh"],
+        "supported_destination_sync_modes": ["overwrite", "append_dedup"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "users/field_type_test",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": "object",
+          "properties": {}
+        },
+        "supported_sync_modes": ["full_refresh"],
+        "supported_destination_sync_modes": ["overwrite", "append_dedup"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "users/50_columns",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": "object",
+          "properties": {}
+        },
+        "supported_sync_modes": ["full_refresh"],
+        "supported_destination_sync_modes": ["overwrite", "append_dedup"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "users/checkboxes",
         "json_schema": {
           "$schema": "http://json-schema.org/draft-07/schema#",
           "type": "object",
@@ -16,7 +72,7 @@
     },
     {
       "stream": {
-        "name": "Table 2",
+        "name": "untitled_base/table_1",
         "json_schema": {
           "$schema": "http://json-schema.org/draft-07/schema#",
           "type": "object",

diff --git a/airbyte-integrations/connectors/source-airtable/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-airtable/integration_tests/invalid_config.json
@@ -1,5 +1,3 @@
 {
-  "api_key": "key####################",
-  "base_id": "app####################",
-  "tables": ["Table 1", "Table 2"]
+  "api_key": "key123456"
 }
diff --git a/airbyte-integrations/connectors/source-airtable/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-airtable/integration_tests/sample_config.json
@@ -1,5 +1,3 @@
 {
-  "api_key": "key1234567890",
-  "base_id": "app1234567890",
-  "tables": ["Table 1", "Table 2"]
+  "api_key": "key1234567890"
 }
diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/helpers.py b/airbyte-integrations/connectors/source-airtable/source_airtable/helpers.py
diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py b/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py
@@ -0,0 +1,44 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+from typing import Any, Dict
+
+from airbyte_cdk.models import AirbyteStream
+from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode
+
+
+class SchemaHelpers:
+    @staticmethod
+    def clean_name(name_str: str) -> str:
+        return name_str.replace(" ", "_").lower().strip()
+
+    @staticmethod
+    def get_json_schema(table: Dict[str, Any]) -> Dict[str, str]:
+        fields = table.get("fields", {})
+        properties = {
+            "_airtable_id": {"type": ["null", "string"]},
+            "_airtable_created_time": {"type": ["null", "string"]},
+        }
+
+        for field in fields:
+            field_name = SchemaHelpers.clean_name(field.get("name"))
+            properties[field_name] = {"type": ["null", "string"]}
+
+        json_schema = {
+            "$schema": "http://json-schema.org/draft-07/schema#",
+            "type": "object",
+            "additionalProperties": True,
+            "properties": properties,
+        }
+        return json_schema
+
+    @staticmethod
+    def get_airbyte_stream(stream_name: str, json_schema: Dict[str, Any]) -> AirbyteStream:
+        return AirbyteStream(
+            name=stream_name,
+            json_schema=json_schema,
+            supported_sync_modes=[SyncMode.full_refresh],
+            supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append_dedup],
+        )