revise incremental strategy

fivetran · Jan 16, 2024 · c145300 · c145300
1 parent 2254104
commit c145300
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 19 deletions.
diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -10,7 +10,7 @@ dispatch:
 
 vars:
   salesforce_marketing_cloud:
-    salesforce_marketing_cloud_schema: sfmc_integration_tests_06
+    salesforce_marketing_cloud_schema: sfmc_integration_tests_01
     salesforce_marketing_cloud_email_identifier: "email_data"
     salesforce_marketing_cloud_event_identifier: "event_data"
     salesforce_marketing_cloud_link_identifier: "link_data"

diff --git a/models/salesforce_marketing_cloud__events_enhanced.sql b/models/salesforce_marketing_cloud__events_enhanced.sql
@@ -16,8 +16,38 @@
 -- events grain with sends and email information added.
 with events as( 
   select
-    *,
-    -- the below list comes from: https://developer.salesforce.com/docs/marketing/marketing-cloud/guide/eventtype.html
+    *
+  from {{ ref('stg_salesforce_marketing_cloud__event') }}
+
+  {% if is_incremental() %}
+    {% if target.type == 'bigquery' %}
+      cast(event_date as date) >= cast(_dbt_max_partition as date)
+    {% else %}
+      where event_date > (select max(event_date) from {{ this }})
+    {% endif %}
+  {% endif %}
+
+), events_enhanced as (
+  select
+    source_relation,
+    batch_id,
+    bounce_category,
+    bounce_type,
+    created_date,
+    event_date,
+    event_type,
+    event_id,
+    modified_date,
+    opt_in_subscriber_key,
+    question,
+    send_id,
+    smtp_code,
+    smtp_reason,
+    subscriber_key,
+    triggered_send_id,
+    unsubscribed_list_id,
+    event_url,
+    -- the below definitions are found here: https://developer.salesforce.com/docs/marketing/marketing-cloud/guide/eventtype.html
     lower(event_type) = 'click' as is_click,
     lower(event_type) = 'deliveredevent' as is_delivered_event,
     lower(event_type) = 'forwardedemail' as is_forward,
@@ -28,15 +58,7 @@ with events as(
     lower(event_type) = 'sent' as is_sent,
     lower(event_type) = 'survey' as is_survey_response,
     lower(event_type) = 'unsubscribe' as is_unsubscribe
-  from {{ ref('stg_salesforce_marketing_cloud__event') }}
-
-  {% if is_incremental() %}
-      {%- call statement('max_event_date', fetch_result=True) -%}
-        select cast(max(event_date) as date) from {{ this }}
-      {%- endcall -%}
-      {%- set max_event_date = load_result('max_event_date')['data'][0][0] -%}
-    where cast(event_date as date) > cast(coalesce('{{ max_event_date }}', '1900-01-01') as date)
-  {% endif %}
+  from events
 
 ), sends as (
   select *
@@ -48,7 +70,7 @@ with events as(
 
 ), joined as (
   select
-    events.*,
+    events_enhanced.*,
     sends.created_date as send_created_date,
     sends.bcc_email,
     sends.from_address,
@@ -59,10 +81,10 @@ with events as(
     emails.email_name,
     emails.subject as email_subject,
     emails.created_date as email_created_date
-  from events
+  from events_enhanced
   left join sends
-    on sends.send_id = events.send_id
-    and sends.source_relation = events.source_relation
+    on sends.send_id = events_enhanced.send_id
+    and sends.source_relation = events_enhanced.source_relation
   left join emails
     on emails.email_id = sends.email_id
     and emails.source_relation = sends.source_relation