Bugfix/remianing sla issues

CHANGELOG.md

@@ -1,3 +1,14 @@
+# dbt_zendesk v0.16.0
+## 🚨 Minor Upgrade 🚨
+Although this update is not a breaking change, it will likely impact the output of the `zendesk__sla_policies` and `zendesk__sla_metrics` models. [PR #154](https://github.com/fivetran/dbt_zendesk/pull/154) includes the following changes:
+
+## Bug Fixes
+- Addresses the potential issue where the `first_reply_time_business_minutes` metric within the `zendesk__ticket_metrics` model would incorrectly calculate the elapsed time when daylight savings occurred. This change involved adjusting a join to reference the difference of two dates as opposed to timestamps. This more accurately applies a cutoff event during daylight savings.
+- Introduction of an additional condition within the `filtered_reply_times` cte of the `int_zendesk__reply_time_combined` model to ensure tickets replied to before any schedule begins and no business minutes have been spent is reserved for **only** the first day the ticket is open. Previously, this condition _could_ be met on days other than the first. This would potentially result in duplicates of `sla_event_id`'s further downstream in the `zendesk__sla_policies` model.
+
+## Under the Hood
+- Addition of integrity and consistency validation tests within integration tests for the `zendesk__sla_policies` and `zendesk__ticket_metrics` models.
+
 # dbt_zendesk v0.15.0
 
 ## 🚨 Minor Upgrade 🚨

README.md

@@ -64,7 +64,7 @@ Include the following zendesk package version in your `packages.yml` file:
 ```yml
 packages:
   - package: fivetran/zendesk
-    version: [">=0.14.0", "<0.15.0"]
+    version: [">=0.16.0", "<0.17.0"]
 
 ```
 > **Note**: Do not include the Zendesk Support source package. The Zendesk Support transform package already has a dependency on the source in its own `packages.yml` file.

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'zendesk'
-version: '0.15.0'
+version: '0.16.0'
 
 
 config-version: 2

integration_tests/dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'zendesk_integration_tests'
-version: '0.15.0'
+version: '0.16.0'
 
 profile: 'integration_tests'
 
@@ -26,6 +26,19 @@ vars:
     zendesk_user_identifier:                   "user_data"
     zendesk_user_tag_identifier:               "user_tag_data"
 
+  ## For validation testing. To be commented out before release.
+  # zendesk_schema: zendesk_test_env
+  # using_domain_names: false
+  # using_user_tags: false
+  # using_organization_tags: false
+  # fivetran_integrity_sla_first_reply_time_exclusion_tickets: (1,56,80)
+  # fivetran_consistency_ticket_metrics_exclusion_tickets: (11092,11093,11094)
+  # fivetran_integrity_sla_count_match_tickets: (76)
+
+
+models:
+  +schema: "zendesk_{{ var('directed_schema','dev') }}"
+
 seeds:
   +quote_columns: "{{ true if target.type == 'redshift' else false }}"
   zendesk_integration_tests:

integration_tests/packages.yml

@@ -1,2 +1,2 @@
 packages:
-  - local: ../
+  - local: ../

integration_tests/requirements.txt

@@ -1,7 +1,7 @@
-dbt-snowflake>=1.3.0,<2.0.0
-dbt-bigquery>=1.3.0,<2.0.0
-dbt-redshift>=1.3.0,<2.0.0
-dbt-postgres>=1.3.0,<2.0.0
-dbt-spark>=1.3.0,<2.0.0
-dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.6.0,<2.0.0
+dbt-snowflake>=1.3.0,<1.8.0
+dbt-bigquery>=1.3.0,<1.8.0
+dbt-redshift>=1.3.0,<1.8.0
+dbt-postgres>=1.3.0,<1.8.0
+dbt-spark>=1.3.0,<1.8.0
+dbt-spark[PyHive]>=1.3.0,<1.8.0
+dbt-databricks>=1.6.0,<1.8.0

integration_tests/tests/consistency/consistency_sla_policies.sql

@@ -0,0 +1,47 @@
+
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select
+        ticket_id,
+        metric, 
+        sla_applied_at,
+        sla_elapsed_time,
+        is_sla_breach
+    from {{ target.schema }}_zendesk_prod.zendesk__sla_policies
+),
+
+dev as (
+    select
+        ticket_id,
+        metric, 
+        sla_applied_at,
+        sla_elapsed_time,
+        is_sla_breach
+    from {{ target.schema }}_zendesk_dev.zendesk__sla_policies
+),
+
+final as (
+    select 
+        prod.ticket_id,
+        prod.metric,
+        prod.sla_applied_at,
+        prod.sla_elapsed_time as prod_sla_elapsed_time,
+        dev.sla_elapsed_time as dev_sla_elapsed_time,
+        prod.is_sla_breach as prod_is_sla_breach,
+        dev.is_sla_breach as dev_is_sla_breach
+    from prod
+    full outer join dev 
+        on dev.ticket_id = prod.ticket_id
+            and dev.metric = prod.metric
+            and dev.sla_applied_at = prod.sla_applied_at
+)
+
+select *
+from final
+where (abs(prod_sla_elapsed_time - dev_sla_elapsed_time) >= 5
+    or prod_is_sla_breach != dev_is_sla_breach)
+    {{ "and prod.ticket_id not in " ~ var('fivetran_consistency_sla_policies_exclusion_tickets',[]) ~ "" if var('fivetran_consistency_sla_policies_exclusion_tickets',[]) }}

integration_tests/tests/consistency/consistency_sla_policy_count.sql

@@ -0,0 +1,35 @@
+
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select
+        1 as join_key,
+        count(*) as total_slas
+    from {{ target.schema }}_zendesk_prod.zendesk__sla_policies
+    group by 1
+),
+
+dev as (
+    select
+        1 as join_key,
+        count(*) as total_slas
+    from {{ target.schema }}_zendesk_dev.zendesk__sla_policies
+    group by 1
+),
+
+final as (
+    select 
+        prod.join_key,
+        prod.total_slas as prod_sla_total,
+        dev.total_slas as dev_sla_total
+    from prod
+    full outer join dev 
+        on dev.join_key = prod.join_key
+)
+
+select *
+from final
+where prod_sla_total != dev_sla_total

integration_tests/tests/consistency/consistency_ticket_metrics.sql

@@ -0,0 +1,39 @@
+
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select
+        ticket_id,
+        first_reply_time_business_minutes, 
+        first_reply_time_calendar_minutes
+    from {{ target.schema }}_zendesk_prod.zendesk__ticket_metrics
+),
+
+dev as (
+    select
+        ticket_id,
+        first_reply_time_business_minutes, 
+        first_reply_time_calendar_minutes
+    from {{ target.schema }}_zendesk_dev.zendesk__ticket_metrics
+),
+
+final as (
+    select 
+        prod.ticket_id,
+        prod.first_reply_time_business_minutes as prod_first_reply_time_business_minutes,
+        dev.first_reply_time_business_minutes as dev_first_reply_time_business_minutes,
+        prod.first_reply_time_calendar_minutes as prod_first_reply_time_calendar_minutes,
+        dev.first_reply_time_calendar_minutes as dev_first_reply_time_calendar_minutes
+    from prod
+    full outer join dev 
+        on dev.ticket_id = prod.ticket_id
+)
+
+select *
+from final
+where (abs(prod_first_reply_time_business_minutes - dev_first_reply_time_business_minutes) >= 5
+    or abs(prod_first_reply_time_calendar_minutes - dev_first_reply_time_calendar_minutes) >= 5)
+    {{ "and ticket_id not in " ~ var('fivetran_consistency_ticket_metrics_exclusion_tickets',[]) ~ "" if var('fivetran_consistency_ticket_metrics_exclusion_tickets',[]) }}

integration_tests/tests/integrity/sla_count_match.sql

@@ -0,0 +1,47 @@
+
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- The necessary source and source_filter adjustments used below originate from the int_zendesk__sla_policy_applied model
+with source as (
+    select
+        *,
+        case when field_name = 'first_reply_time' then row_number() over (partition by ticket_id, field_name order by valid_starting_at desc) else 1 end as latest_sla
+    from {{ ref('stg_zendesk__ticket_field_history') }}
+),
+
+source_filter as (
+    select
+        ticket_id,
+        count(*) as source_row_count
+    from source
+    where field_name in ('next_reply_time', 'first_reply_time', 'agent_work_time', 'requester_wait_time')
+        and value is not null
+        and latest_sla = 1
+    group by 1
+),
+
+sla_policies as (
+    select
+        ticket_id,
+        count(*) as end_model_row_count
+    from {{ ref('zendesk__sla_policies') }}
+    group by 1
+),
+
+match_check as (
+    select 
+        sla_policies.ticket_id,
+        end_model_row_count,
+        source_row_count
+    from sla_policies
+    full outer join source_filter
+        on source_filter.ticket_id = sla_policies.ticket_id
+)
+
+select *
+from match_check
+where end_model_row_count != source_row_count
+{{ "and ticket_id not in " ~ var('fivetran_integrity_sla_count_match_tickets',[]) ~ "" if var('fivetran_integrity_sla_count_match_tickets',[]) }}

integration_tests/tests/integrity/sla_first_reply_time_match.sql

@@ -0,0 +1,36 @@
+
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with ticket_metrics as (
+    select
+        ticket_id,
+        first_reply_time_business_minutes
+    from {{ ref('zendesk__ticket_metrics') }}
+),
+
+sla_policies as (
+    select
+        ticket_id,
+        sla_elapsed_time
+    from {{ ref('zendesk__sla_policies') }}
+    where metric = 'first_reply_time'
+        and in_business_hours
+),
+
+match_check as (
+    select 
+        ticket_metrics.ticket_id,
+        ticket_metrics.first_reply_time_business_minutes,
+        sla_policies.sla_elapsed_time
+    from ticket_metrics
+    full outer join sla_policies 
+        on ticket_metrics.ticket_id = sla_policies.ticket_id
+)
+
+select *
+from match_check
+where abs(round(first_reply_time_business_minutes,0) - round(sla_elapsed_time,0)) >= 2
+    {{ "and ticket_id not in " ~ var('fivetran_integrity_sla_first_reply_time_exclusion_tickets',[]) ~ "" if var('fivetran_integrity_sla_first_reply_time_exclusion_tickets',[]) }}

models/reply_times/int_zendesk__ticket_first_reply_time_business.sql

@@ -96,8 +96,8 @@ with ticket_reply_times as (
     and weekly_periods.schedule_id = schedule.schedule_id
       -- this chooses the Daylight Savings Time or Standard Time version of the schedule
       -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week
-    and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }})
-    and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) < cast(schedule.valid_until as {{ dbt.type_timestamp() }})
+    and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as date) > cast(schedule.valid_from as date)
+    and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time', from_date_or_timestamp='start_week_date') }} as date) < cast(schedule.valid_until as date)
 
 )
 

models/sla_policy/reply_time/int_zendesk__reply_time_combined.sql

@@ -110,6 +110,7 @@ with reply_time_calendar_hours_sla as (
 ), lagging_time_block as (
   select
     *,
+    row_number() over (partition by ticket_id, metric, sla_applied_at order by sla_schedule_start_at) as day_index,
     lead(sla_schedule_start_at) over (partition by ticket_id, sla_policy_name, metric, sla_applied_at order by sla_schedule_start_at) as next_schedule_start,
     min(sla_breach_at) over (partition by sla_policy_name, metric, sla_applied_at order by sla_schedule_start_at rows unbounded preceding) as first_sla_breach_at,
 		coalesce(lag(sum_lapsed_business_minutes) over (partition by sla_policy_name, metric, sla_applied_at order by sla_schedule_start_at), 0) as sum_lapsed_business_minutes_new,
@@ -124,7 +125,7 @@ with reply_time_calendar_hours_sla as (
     in_business_hours
       and ((
         agent_reply_at >= sla_schedule_start_at and agent_reply_at <= sla_schedule_end_at) -- ticket is replied to between a schedule window
-        or (agent_reply_at < sla_schedule_start_at and sum_lapsed_business_minutes_new = 0 and sla_breach_at = first_sla_breach_at) -- ticket is replied to before a schedule window and no business minutes have been spent on it
+        or (agent_reply_at < sla_schedule_start_at and sum_lapsed_business_minutes_new = 0 and sla_breach_at = first_sla_breach_at and day_index = 1) -- ticket is replied to before any schedule begins and no business minutes have been spent on it
         or (agent_reply_at is null and next_solved_at >= sla_schedule_start_at and next_solved_at < next_schedule_start) -- There are no reply times, but the ticket is closed and we should capture the closed date as the first and/or next reply time if there is not one preceding.
         or (next_solved_at is null and agent_reply_at is null and {{ dbt.current_timestamp() }} >= sla_schedule_start_at and ({{ dbt.current_timestamp() }} < next_schedule_start or next_schedule_start is null)) -- ticket is not replied to and therefore active. But only bring through the active SLA record that is most recent (after the last SLA schedule starts but before the next, or if there does not exist a next SLA schedule start time)  
         or (agent_reply_at > sla_schedule_end_at and (agent_reply_at < next_schedule_start or next_schedule_start is null)) -- ticket is replied to outside sla schedule hours