🐛 🎉 Source Airtable: migrate to the Metadata API for dynamic schema generation

airbyte-integrations/connectors/source-airtable/Dockerfile

@@ -34,5 +34,5 @@ COPY source_airtable ./source_airtable
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.3
+LABEL io.airbyte.version=1.0.0
 LABEL io.airbyte.name=airbyte/source-airtable

airbyte-integrations/connectors/source-airtable/README.md

@@ -103,7 +103,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce
 If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
 To run your integration tests with acceptance tests, from the connector root, run
 ```
-python -m pytest integration_tests -p integration_tests.acceptance
+docker build . --no-cache -t airbyte/source-airtable:dev \
+&& python -m pytest integration_tests -p integration_tests.acceptance
 ```
 To run your integration tests with docker
 

airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml

@@ -1,20 +1,28 @@
 # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
 # for more information about how to configure these tests
 connector_image: airbyte/source-airtable:dev
-tests:
+acceptance_tests:
   spec:
-    - spec_path: "source_airtable/spec.json"
+    tests:
+      - spec_path: "source_airtable/spec.json"
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
+    tests:
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
   discovery:
-    - config_path: "secrets/config.json"
+    tests:
+      - config_path: "secrets/config.json"
+      # bypassed this check, because discovery mechanism was changed
+        backward_compatibility_tests_config:
+          disable_for_version: "0.1.3"
   basic_read:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      empty_streams: []
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        empty_streams: []
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"

airbyte-integrations/connectors/source-airtable/integration_tests/configured_catalog.json

@@ -2,7 +2,63 @@
   "streams": [
     {
       "stream": {
-        "name": "Table 1",
+        "name": "users/table_1",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": "object",
+          "properties": {}
+        },
+        "supported_sync_modes": ["full_refresh"],
+        "supported_destination_sync_modes": ["overwrite", "append_dedup"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    }, 
+    {
+      "stream": {
+        "name": "users/table_2",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": "object",
+          "properties": {}
+        },
+        "supported_sync_modes": ["full_refresh"],
+        "supported_destination_sync_modes": ["overwrite", "append_dedup"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "users/field_type_test",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": "object",
+          "properties": {}
+        },
+        "supported_sync_modes": ["full_refresh"],
+        "supported_destination_sync_modes": ["overwrite", "append_dedup"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "users/50_columns",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": "object",
+          "properties": {}
+        },
+        "supported_sync_modes": ["full_refresh"],
+        "supported_destination_sync_modes": ["overwrite", "append_dedup"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "users/checkboxes",
         "json_schema": {
           "$schema": "http://json-schema.org/draft-07/schema#",
           "type": "object",
@@ -16,7 +72,7 @@
     },
     {
       "stream": {
-        "name": "Table 2",
+        "name": "untitled_base/table_1",
         "json_schema": {
           "$schema": "http://json-schema.org/draft-07/schema#",
           "type": "object",

airbyte-integrations/connectors/source-airtable/integration_tests/invalid_config.json

@@ -1,5 +1,3 @@
 {
-  "api_key": "key####################",
-  "base_id": "app####################",
-  "tables": ["Table 1", "Table 2"]
+  "api_key": "key123456"
 }

airbyte-integrations/connectors/source-airtable/integration_tests/sample_config.json

@@ -1,5 +1,3 @@
 {
-  "api_key": "key1234567890",
-  "base_id": "app1234567890",
-  "tables": ["Table 1", "Table 2"]
+  "api_key": "key1234567890"
 }

airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py

@@ -0,0 +1,60 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+from typing import Any, Dict, Iterable, Mapping
+
+from airbyte_cdk.models import AirbyteStream
+from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode
+
+
+class SchemaHelpers:
+
+    streams_catalog: Iterable[Mapping[str, Any]] = []
+
+    def get_streams_from_base(self, base_id: str, base_name: str, base_tables: list):
+        for table in base_tables:
+            if table not in self.streams_catalog:
+                self.streams_catalog.append(
+                    {
+                        "stream_path": f"{base_id}/{table.get('id')}",
+                        "stream": self.get_airbyte_stream(
+                            f"{base_name}/{self.clean_name(table.get('name'))}",
+                            self.get_json_schema(table),
+                        ),
+                    }
+                )
+
+    @staticmethod
+    def clean_name(name_str: str) -> str:
+        return name_str.replace(" ", "_").lower().strip()
+
+    @staticmethod
+    def get_json_schema(table: Dict[str, Any]) -> Dict[str, str]:
+        fields = table.get("fields", {})
+        properties = {
+            "_airtable_id": {"type": ["null", "string"]},
+            "_airtable_created_time": {"type": ["null", "string"]},
+        }
+
+        for field in fields:
+            field_name = SchemaHelpers.clean_name(field.get("name"))
+            properties[field_name] = {"type": ["null", "string"]}
+
+        json_schema = {
+            "$schema": "http://json-schema.org/draft-07/schema#",
+            "type": "object",
+            "additionalProperties": True,
+            "properties": properties,
+        }
+        return json_schema
+
+    @staticmethod
+    def get_airbyte_stream(stream_name: str, json_schema: Dict[str, Any]) -> AirbyteStream:
+        return AirbyteStream(
+            name=stream_name,
+            json_schema=json_schema,
+            supported_sync_modes=[SyncMode.full_refresh],
+            supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append_dedup],
+        )