From 4f5c1a4c7cb051be1db7485a3a18f3d74ef2ebd1 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 2 Jan 2024 15:39:58 -0600 Subject: [PATCH 1/3] review/jm-initial --- .github/workflows/auto-release.yml | 2 +- dbt_project.yml | 2 +- .../salesforce_marketing_cloud__events_enhanced.sql | 4 ++-- .../salesforce_marketing_cloud__subscribers_enhanced.sql | 2 +- models/salesforce_marketing_cloud__email_overview.sql | 2 +- models/salesforce_marketing_cloud__sends_links.sql | 6 +++--- models/staging/stg_salesforce_marketing_cloud__email.sql | 6 +++--- models/staging/stg_salesforce_marketing_cloud__event.sql | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index d073ebf..8ed5853 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -9,5 +9,5 @@ on: jobs: call-workflow-passing-data: if: github.event.pull_request.merged - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@feature/auto-releaser + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main secrets: inherit \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 41a46c2..e37ac5b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -9,7 +9,7 @@ models: +materialized: table intermediate: +materialized: ephemeral - +schema: int_sfmc + +schema: int_sfmc staging: +materialized: view +schema: stg_sfmc diff --git a/models/intermediate/salesforce_marketing_cloud__events_enhanced.sql b/models/intermediate/salesforce_marketing_cloud__events_enhanced.sql index c825de9..9a4ec3a 100644 --- a/models/intermediate/salesforce_marketing_cloud__events_enhanced.sql +++ b/models/intermediate/salesforce_marketing_cloud__events_enhanced.sql @@ -1,10 +1,10 @@ -{{ +{{ config( materialized='incremental', unique_key='event_id', partition_by={'field': 'event_date', 'data_type': 'date'} if target.type not in ('spark','databricks') else ['event_date'], cluster_by=['event_date'], - incremental_strategy = 'merge' if target.type not in ('postgres', 'redshift') else 'delete+insert', + incremental_strategy = 'merge' if target.type not in ('postgres', 'redshift') else 'delete+insert', file_format = 'delta' ) }} diff --git a/models/intermediate/salesforce_marketing_cloud__subscribers_enhanced.sql b/models/intermediate/salesforce_marketing_cloud__subscribers_enhanced.sql index d72a84b..798b28a 100644 --- a/models/intermediate/salesforce_marketing_cloud__subscribers_enhanced.sql +++ b/models/intermediate/salesforce_marketing_cloud__subscribers_enhanced.sql @@ -1,4 +1,4 @@ --- this is meant to be stand alone since subscribers could belong to multiple lists, and I did not want to do a list agg to include this in the subscriber overview, but open for discussion +-- this is meant to be stand alone since subscribers could belong to multiple lists, and I did not want to do a list agg to include this in the subscriber overview, but open for discussion with subscribers as ( select *, diff --git a/models/salesforce_marketing_cloud__email_overview.sql b/models/salesforce_marketing_cloud__email_overview.sql index 3b07409..d38321f 100644 --- a/models/salesforce_marketing_cloud__email_overview.sql +++ b/models/salesforce_marketing_cloud__email_overview.sql @@ -7,7 +7,7 @@ with emails as ( ), sends as ( select * from {{ ref('stg_salesforce_marketing_cloud__send') }} - where not _fivetran_deleted + where not _fivetran_deleted ), sends_stats as ( select diff --git a/models/salesforce_marketing_cloud__sends_links.sql b/models/salesforce_marketing_cloud__sends_links.sql index b2e20cc..9b53d25 100644 --- a/models/salesforce_marketing_cloud__sends_links.sql +++ b/models/salesforce_marketing_cloud__sends_links.sql @@ -4,15 +4,15 @@ with sends as ( select * from {{ ref('stg_salesforce_marketing_cloud__send') }} - where not _fivetran_deleted + where not _fivetran_deleted ), link_sends as ( select * - from {{ ref('stg_salesforce_marketing_cloud__link_send') }} + from {{ ref('stg_salesforce_marketing_cloud__link_send') }} ), links as ( select * - from {{ ref('stg_salesforce_marketing_cloud__link') }} + from {{ ref('stg_salesforce_marketing_cloud__link') }} ), joined as ( select diff --git a/models/staging/stg_salesforce_marketing_cloud__email.sql b/models/staging/stg_salesforce_marketing_cloud__email.sql index 1f19aba..4b71e76 100644 --- a/models/staging/stg_salesforce_marketing_cloud__email.sql +++ b/models/staging/stg_salesforce_marketing_cloud__email.sql @@ -26,9 +26,9 @@ final as ( select source_relation, - _fivetran_start, - _fivetran_end, - _fivetran_active, + _fivetran_start, + _fivetran_end, + _fivetran_active, _fivetran_synced, asset_id, asset_type_id, diff --git a/models/staging/stg_salesforce_marketing_cloud__event.sql b/models/staging/stg_salesforce_marketing_cloud__event.sql index 5de3ac2..696e677 100644 --- a/models/staging/stg_salesforce_marketing_cloud__event.sql +++ b/models/staging/stg_salesforce_marketing_cloud__event.sql @@ -26,7 +26,7 @@ final as ( select source_relation, - _fivetran_deleted, + _fivetran_deleted, _fivetran_synced, batch_id, bounce_category, From 35255e416db49b004ff821a00ce5f5c28f477adf Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 3 Jan 2024 10:55:14 -0600 Subject: [PATCH 2/3] additional notes --- README.md | 14 +++++++------- dbt_project.yml | 2 +- .../salesforce_marketing_cloud__email_overview.sql | 10 +++++----- .../salesforce_marketing_cloud__sends_overview.sql | 6 +++--- ...alesforce_marketing_cloud__subscriber_lists.sql | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index f1ab564..377f233 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ This package models Salesforce Marketing Cloud data from [Fivetran's connector]( The main focus of the package is to transform the core object tables into analytics-ready models: - - Materializes [Salesforce Marketing Cloud staging tables](https://fivetran.github.io/dbt_salesforce_marketing_cloud/#!/overview/salesforce_marketing_cloud_source/models/?g_v=1) which leverage data in the format described by [this ERD](https://fivetran.com/docs/applications/salesforce_marketing_cloud/#schemainformation). The staging tables clean, test, and prepare your Salesforce Marketing Cloud data from [Fivetran's connector](https://fivetran.com/docs/applications/salesforce_marketing_cloud_source) for analysis by doing the following: + - Materializes [Salesforce Marketing Cloud staging tables](https://fivetran.github.io/dbt_salesforce_marketing_cloud/#!/overview/salesforce_marketing_cloud_source/models/?g_v=1) which leverage data in the format described by [this ERD](https://fivetran.com/docs/applications/salesforce_marketing_cloud/#schemainformation). The staging tables clean, test, and prepare your Salesforce Marketing Cloud data from [Fivetran's connector](https://fivetran.com/docs/applications/salesforce_marketing_cloud) for analysis by doing the following: - Primary keys are renamed from `id` to `_id`. - Adds column-level testing where applicable. For example, all primary keys are tested for uniqueness and non-null values. - Provides insight into your Salesforce Marketing Cloud data across the following grains: @@ -28,15 +28,15 @@ The main focus of the package is to transform the core object tables into analyt The following table provides a detailed list of all models materialized within this package by default. > [!TIP] -> See more details about these models in the package's [dbt docs site](https://fivetran.github.io/dbt_salesforce_marketing_cloud/#!/overview/qualtrics). +> See more details about these models in the package's [dbt docs site](https://fivetran.github.io/dbt_salesforce_marketing_cloud/#!/overview). | **model** | **description**| | --------- | -------------- | -| [salesforce_marketing_cloud__email_overview]() | Each record represents an email with aggregated send and event data for each. | -| [salesforce_marketing_cloud__sends_links]() | Each record represents a link and the corresponding send(s). | -| [salesforce_marketing_cloud__sends_overview]() | Each record represents a send with aggregated event data for each. | -| [salesforce_marketing_cloud__subscriber_lists]() | Each record represents a list and the corresponding subscriber(s). | -| [salesforce_marketing_cloud__subscriber_overview]() | Each record represents a subscriber with aggregated event data for each. | +| [salesforce_marketing_cloud__email_overview]() | Each record represents an email with aggregated send and event data for each. | +| [salesforce_marketing_cloud__sends_links]() | Each record represents a link and the corresponding send(s). | +| [salesforce_marketing_cloud__sends_overview]() | Each record represents a send with aggregated event data for each. | +| [salesforce_marketing_cloud__subscriber_lists]() | Each record represents a list and the corresponding subscriber(s). | +| [salesforce_marketing_cloud__subscriber_overview]() | Each record represents a subscriber with aggregated event data for each. | # 🎯 How do I use the dbt package? diff --git a/dbt_project.yml b/dbt_project.yml index e37ac5b..6b85858 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -9,7 +9,7 @@ models: +materialized: table intermediate: +materialized: ephemeral - +schema: int_sfmc + +schema: int_sfmc staging: +materialized: view +schema: stg_sfmc diff --git a/models/salesforce_marketing_cloud__email_overview.sql b/models/salesforce_marketing_cloud__email_overview.sql index d38321f..f553430 100644 --- a/models/salesforce_marketing_cloud__email_overview.sql +++ b/models/salesforce_marketing_cloud__email_overview.sql @@ -31,9 +31,9 @@ with emails as ( ), sends_aggs as ( select sends_stats.*, - coalesce(total_unique_opens / nullif(total_emails_sent, 0), 0) as open_rate, --use safe divide in dbt - coalesce(total_unique_clicks / nullif(total_emails_sent, 0), 0) as click_through_rate, --use safe divide in dbt - coalesce(total_unsubscribes / nullif(total_emails_sent, 0), 0) as unsubscribe_rate --use safe divide in dbt + coalesce(total_unique_opens / nullif(total_emails_sent, 0), 0) as open_rate, --use safe divide in dbt + coalesce(total_unique_clicks / nullif(total_emails_sent, 0), 0) as click_through_rate, --use safe divide in dbt + coalesce(total_unsubscribes / nullif(total_emails_sent, 0), 0) as unsubscribe_rate --use safe divide in dbt from sends_stats ), events_enhanced as ( @@ -72,8 +72,8 @@ with emails as ( sends_aggs.total_deliveries, sends_aggs.total_targets, sends_aggs.total_bounces, - sends_aggs.open_rate, --use safe divide in dbt - sends_aggs.click_through_rate, --use safe divide in dbt + sends_aggs.open_rate, --use safe divide in dbt + sends_aggs.click_through_rate, --use safe divide in dbt sends_aggs.unsubscribe_rate, events_stats.total_send_events, events_stats.total_open_events, diff --git a/models/salesforce_marketing_cloud__sends_overview.sql b/models/salesforce_marketing_cloud__sends_overview.sql index 1d697dc..2c2df8f 100644 --- a/models/salesforce_marketing_cloud__sends_overview.sql +++ b/models/salesforce_marketing_cloud__sends_overview.sql @@ -7,9 +7,9 @@ with sends as ( ), sends_aggs as ( select sends.*, - coalesce(unique_opens / nullif(number_sent, 0), 0) as open_rate, --use safe divide in dbt - coalesce(unique_clicks / nullif(number_sent, 0), 0) as click_through_rate, --use safe divide in dbt - coalesce(unsubscribes / nullif(number_sent, 0), 0) as unsubscribe_rate --use safe divide in dbt + coalesce(unique_opens / nullif(number_sent, 0), 0) as open_rate, --use safe divide in dbt + coalesce(unique_clicks / nullif(number_sent, 0), 0) as click_through_rate, --use safe divide in dbt + coalesce(unsubscribes / nullif(number_sent, 0), 0) as unsubscribe_rate --use safe divide in dbt from sends ), events_enhanced as ( diff --git a/models/salesforce_marketing_cloud__subscriber_lists.sql b/models/salesforce_marketing_cloud__subscriber_lists.sql index 6c3cc5a..8f926d5 100644 --- a/models/salesforce_marketing_cloud__subscriber_lists.sql +++ b/models/salesforce_marketing_cloud__subscriber_lists.sql @@ -10,7 +10,7 @@ with subscribers as ( from {{ ref('stg_salesforce_marketing_cloud__list_subscriber') }} where coalesce(_fivetran_active, true) -), lists as ( --make lists optional +), lists as ( --make lists optional select * from {{ ref('stg_salesforce_marketing_cloud__list') }} where coalesce(_fivetran_active, true) From f3c0ac64befde8025a6807c2c750072ecf749147 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:07:43 -0600 Subject: [PATCH 3/3] final notes --- dbt_project.yml | 3 ++- models/salesforce_marketing_cloud__email_overview.sql | 2 +- models/salesforce_marketing_cloud__sends_overview.sql | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 6b85858..dff6f54 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -9,13 +9,14 @@ models: +materialized: table intermediate: +materialized: ephemeral - +schema: int_sfmc + +schema: int_sfmc staging: +materialized: view +schema: stg_sfmc vars: + salesforce_marketing_cloud: email: "{{ source('salesforce_marketing_cloud','email') }}" event: "{{ source('salesforce_marketing_cloud','event') }}" link: "{{ source('salesforce_marketing_cloud','link') }}" diff --git a/models/salesforce_marketing_cloud__email_overview.sql b/models/salesforce_marketing_cloud__email_overview.sql index f553430..59de3bb 100644 --- a/models/salesforce_marketing_cloud__email_overview.sql +++ b/models/salesforce_marketing_cloud__email_overview.sql @@ -62,7 +62,7 @@ with emails as ( ), joined as ( select - emails.*, + emails.*, sends_aggs.total_unique_sends, sends_aggs.total_emails_sent, sends_aggs.total_unique_opens, diff --git a/models/salesforce_marketing_cloud__sends_overview.sql b/models/salesforce_marketing_cloud__sends_overview.sql index 2c2df8f..04bc749 100644 --- a/models/salesforce_marketing_cloud__sends_overview.sql +++ b/models/salesforce_marketing_cloud__sends_overview.sql @@ -35,7 +35,7 @@ with sends as ( ), joined as ( select - sends_aggs.*, + sends_aggs.*, events_stats.total_send_events, events_stats.total_open_events, events_stats.total_click_events,