Source Mixpanel: SAT added incremental testing for "engage" stream (#…

…17300) Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
airbytehq · Sep 28, 2022 · e8736ee · e8736ee
1 parent fe2238f
commit e8736ee
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 80 deletions.
diff --git a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml
@@ -31,7 +31,7 @@ tests:
       timeout_seconds: 3600
   incremental:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog_incremental.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
       future_state_path: "integration_tests/abnormal_state.json"
       cursor_paths:
         cohorts: ["created"]

diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json
@@ -15,5 +15,8 @@
   },
   "cohort_members": {
     "last_seen": "2030-01-01T00:00:00"
+  },
+  "engage": {
+    "last_seen": "2030-01-01T00:00:00"
   }
 }
diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog.json
@@ -5,19 +5,25 @@
         "name": "funnels",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
-        "default_cursor_field": ["date"]
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date"],
+        "source_defined_primary_key": [["funnel_id"], ["date"]]
       },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append"
     },
     {
       "stream": {
         "name": "engage",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": false,
+        "default_cursor_field": null,
+        "source_defined_primary_key": [["distinct_id"]]
       },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "cursor_field": ["last_seen"]
     },
     {
       "stream": {
@@ -33,37 +39,47 @@
         "name": "export",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
         "default_cursor_field": ["time"]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
       "destination_sync_mode": "append"
     },
     {
       "stream": {
         "name": "cohorts",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"]
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created"],
+        "source_defined_primary_key": [["id"]]
       },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append"
     },
     {
       "stream": {
         "name": "cohort_members",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh", "incremental"]
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": false,
+        "default_cursor_field": null,
+        "source_defined_primary_key": [["distinct_id"]]
       },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "overwrite"
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append",
+      "cursor_field": ["last_seen"]
     },
     {
       "stream": {
         "name": "revenue",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
-        "default_cursor_field": ["date"]
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date"],
+        "source_defined_primary_key": [["date"]]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
       "destination_sync_mode": "append"
     }
   ]

diff --git a/...grations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json b/...grations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json