From 4010993a0e3247615d1f021986fb1f8779b5e66b Mon Sep 17 00:00:00 2001 From: Zoe Howard Date: Fri, 28 Oct 2022 11:20:26 -0400 Subject: [PATCH 1/5] create zendesk replies model --- .../int_zendesk__ticket_reply_times.sql | 4 +- models/zendesk__reply_metrics.sql | 94 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 models/zendesk__reply_metrics.sql diff --git a/models/reply_times/int_zendesk__ticket_reply_times.sql b/models/reply_times/int_zendesk__ticket_reply_times.sql index ba9c633c..46e6717c 100644 --- a/models/reply_times/int_zendesk__ticket_reply_times.sql +++ b/models/reply_times/int_zendesk__ticket_reply_times.sql @@ -8,6 +8,7 @@ with ticket_public_comments as ( select ticket_id, + user_id, valid_starting_at as end_user_comment_created_at, ticket_created_date, commenter_role, @@ -30,7 +31,8 @@ with ticket_public_comments as ( and end_user_comments.commenter_role != 'external_comment' and (end_user_comments.previous_internal_comment_count > 0) then end_user_comments.end_user_comment_created_at - else agent_comments.valid_starting_at end) as agent_responded_at + else agent_comments.valid_starting_at end) as agent_responded_at, + min(agent_comments.user_id) as responding_agent_user_id from end_user_comments left join ticket_public_comments as agent_comments on agent_comments.ticket_id = end_user_comments.ticket_id diff --git a/models/zendesk__reply_metrics.sql b/models/zendesk__reply_metrics.sql new file mode 100644 index 00000000..967c35d6 --- /dev/null +++ b/models/zendesk__reply_metrics.sql @@ -0,0 +1,94 @@ +with commenter as ( + + select * from {{ ref('stg_zendesk__user') }} + +), ticket_reply_times as ( + + select * from {{ ref('int_zendesk__ticket_reply_times') }} + +), ticket_schedules as ( + + select * from {{ ref('int_zendesk__ticket_schedules') }} + +), schedule as ( + + select * from {{ ref('int_zendesk__schedule_spine') }} + +), replies as ( + + select ticket_reply_times.ticket_id + , ticket_reply_times.end_user_comment_created_at + , ticket_reply_times.agent_responded_at + , ticket_reply_times.responding_agent_user_id + , ticket_reply_times.reply_time_calendar_minutes + , ticket_schedules.schedule_created_at + , ticket_schedules.schedule_invalidated_at + , ticket_schedules.schedule_id + + , ({{ fivetran_utils.timestamp_diff( + "cast(" ~ dbt_date.week_start('ticket_reply_times.end_user_comment_created_at','UTC') ~ "as " ~ dbt_utils.type_timestamp() ~ ")", + "cast(ticket_reply_times.end_user_comment_created_at as " ~ dbt_utils.type_timestamp() ~ ")", + 'second') }} / 60.0 + ) as start_time_in_minutes_from_week + + from ticket_reply_times + join ticket_schedules on ticket_reply_times.ticket_id = ticket_schedules.ticket_id + group by 1,2,3,4,5,6,7,8 + +), weeks as ( + + {{ dbt_utils.generate_series(208) }} + +), weeks_cross_ticket_reply_times as ( + -- because time is reported in minutes since the beginning of the week, we have to split up time spent on the ticket into calendar weeks + select replies.* + , generated_number - 1 as week_number + + from replies + cross join weeks + where floor((start_time_in_minutes_from_week + reply_time_calendar_minutes) / (7*24*60)) >= generated_number - 1 + +), weekly_periods as ( + + select weeks_cross_ticket_reply_times.* + , greatest(0, start_time_in_minutes_from_week - week_number * (7*24*60)) as ticket_week_start_time + , least(start_time_in_minutes_from_week + reply_time_calendar_minutes - week_number * (7*24*60), (7*24*60)) as ticket_week_end_time + + from weeks_cross_ticket_reply_times + +), intercepted_periods as ( + + select weekly_periods.* + , schedule.start_time_utc as schedule_start_time + , schedule.end_time_utc as schedule_end_time + , least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes + + from weekly_periods + join schedule on ticket_week_start_time <= schedule.end_time_utc + and ticket_week_end_time >= schedule.start_time_utc + and weekly_periods.schedule_id = schedule.schedule_id + -- this chooses the Daylight Savings Time or Standard Time version of the schedule + and weekly_periods.agent_responded_at >= cast(schedule.valid_from as {{ dbt_utils.type_timestamp() }}) + and weekly_periods.agent_responded_at < cast(schedule.valid_until as {{ dbt_utils.type_timestamp() }}) + +), aggregated as ( + + select ticket_id + , end_user_comment_created_at + , agent_responded_at + , responding_agent_user_id + , reply_time_calendar_minutes + , sum(scheduled_minutes) as reply_time_business_minutes + + from intercepted_periods + group by 1,2,3,4,5 + +) + + select {{ dbt_utils.surrogate_key(['ticket_id', 'end_user_comment_created_at']) }} as reply_id + , aggregated.* + , commenter.name as responding_agent_name + , commenter.email as responding_agent_email + from aggregated + left join commenter + on commenter.user_id = aggregated.responding_agent_user_id \ No newline at end of file From 544f2e763e28ba11ac98017d88a7a92c21aa6bf7 Mon Sep 17 00:00:00 2001 From: Zoe Howard Date: Fri, 28 Oct 2022 11:37:34 -0400 Subject: [PATCH 2/5] Add documentation for new model --- models/zendesk.yml | 22 +++++++++++++++++++++- models/zendesk__reply_metrics.sql | 3 +-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/models/zendesk.yml b/models/zendesk.yml index 5fc509e0..8b232ac5 100644 --- a/models/zendesk.yml +++ b/models/zendesk.yml @@ -547,4 +547,24 @@ models: - name: assignee_name description: The assignee name assigned to the ticket - name: priority - description: The tickets priority ranking \ No newline at end of file + description: The tickets priority ranking + + - name: zendesk__reply_metrics + description: Each record represents an end user comment and the responding agent reply time + columns: + - name: ticket_id + description: A ticket's unique identifier, it is automatically assigned when the ticket is created + - name: end_user_comment_created_at + description: The time the end user created the comment + - name: agent_responded_at + description: Time the agent reponded to that end user comment + - name: responding_agent_user_id + description: User ID of responding agent + - name: reply_time_calendar_minutes + description: Time from end user comment created to agent response in calendar minutes + - name: reply_time_business_minutes + description: Time from end user comment created to agent response in business minutes + - name: responding_agent_name + description: Name of responding agent + - name: responding_agent_email + description: Email of responding agent \ No newline at end of file diff --git a/models/zendesk__reply_metrics.sql b/models/zendesk__reply_metrics.sql index 967c35d6..f5092f12 100644 --- a/models/zendesk__reply_metrics.sql +++ b/models/zendesk__reply_metrics.sql @@ -85,8 +85,7 @@ with commenter as ( ) - select {{ dbt_utils.surrogate_key(['ticket_id', 'end_user_comment_created_at']) }} as reply_id - , aggregated.* + select aggregated.* , commenter.name as responding_agent_name , commenter.email as responding_agent_email from aggregated From 0bb149ec5761dc3d1d846cd9618e2ad6505ca547 Mon Sep 17 00:00:00 2001 From: Zoe Howard Date: Fri, 28 Oct 2022 11:54:23 -0400 Subject: [PATCH 3/5] update CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04146c4d..65b256f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ # dbt_zendesk v0.9.1 + +## Create zendesk__reply_metrics model +- Creates a new end model called `zendesk__reply_metrics` to surface data at the reply granularity, which none of the existing end models do. It is a modified version of the `int_zendesk__ticket_first_reply_time_business` model. +- There are minimal changes to existing models, just including the `user_id` field in the `int_zendesk__ticket_reply_times` model +## Contributiors +- [zhoward101](https://github.com/zhoward101) + ## Bugfix: - If doing a _dbt_compile_ prior to _dbt_run_, it fails at `int_zendesk__calendar_spine` because the staging model it references is not built yet. This PR changes the intermediate models to reference source tables instead of staging models. ([#79](https://github.com/fivetran/dbt_zendesk/pull/79)) ## Contributors From 0fdb6f5d4f6c6d254068bfedbc6cb18487200766 Mon Sep 17 00:00:00 2001 From: Zoe Howard Date: Fri, 28 Oct 2022 12:06:21 -0400 Subject: [PATCH 4/5] update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bd81c5c3..76ad7778 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ The following table provides a detailed list of final models materialized within | **model** | **description** | | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [zendesk__reply_metrics](https://fivetran.github.io/dbt_zendesk/#!/model/model.zendesk.zendesk__reply_metrics) | Each record represents a communication between an end user and a responding agent, enriched with metrics about reply times. Calendar and business hours are supported. | | [zendesk__ticket_metrics](https://fivetran.github.io/dbt_zendesk/#!/model/model.zendesk.zendesk__ticket_metrics) | Each record represents a Zendesk ticket, enriched with metrics about reply times, resolution times, and work times. Calendar and business hours are supported. | | [zendesk__ticket_enriched](https://fivetran.github.io/dbt_zendesk/#!/model/model.zendesk.zendesk__ticket_enriched) | Each record represents a Zendesk ticket, enriched with data about its tags, assignees, requester, submitter, organization, and group. | | [zendesk__ticket_summary](https://fivetran.github.io/dbt_zendesk/#!/model/model.zendesk.zendesk__ticket_summary) | A single record table containing Zendesk ticket and user summary metrics. | From 70d375a0cf32101baea3b43eba738d2d67ffedf1 Mon Sep 17 00:00:00 2001 From: Zoe Howard Date: Sun, 12 Feb 2023 14:08:32 -0500 Subject: [PATCH 5/5] Debugging updates --- .../int_zendesk__comments_enriched.sql | 8 +++- .../int_zendesk__ticket_reply_times.sql | 47 ++++++++++++------- models/zendesk.yml | 12 ++--- models/zendesk__reply_metrics.sql | 31 +++++------- 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/models/reply_times/int_zendesk__comments_enriched.sql b/models/reply_times/int_zendesk__comments_enriched.sql index 34313cf9..cf6a3341 100644 --- a/models/reply_times/int_zendesk__comments_enriched.sql +++ b/models/reply_times/int_zendesk__comments_enriched.sql @@ -14,9 +14,13 @@ with ticket_comment as ( select ticket_comment.*, + commenter.name as commenter_name, + commenter.email as commenter_email, case when commenter.role = 'end-user' then 'external_comment' when commenter.role in ('agent','admin') then 'internal_comment' else 'unknown' end as commenter_role + -- For some reason some of the tickets started with voicemails do not have the voicemail recorded as a public comment? + , (is_public is true or value like 'Voicemail from%') as is_public_comment from ticket_comment @@ -35,7 +39,7 @@ with ticket_comment as ( , 'first_comment') as previous_commenter_role from joined - where is_public + where is_public_comment union all @@ -43,7 +47,7 @@ with ticket_comment as ( *, 'non_public_comment' as previous_commenter_role from joined - where not is_public + where not is_public_comment ) select diff --git a/models/reply_times/int_zendesk__ticket_reply_times.sql b/models/reply_times/int_zendesk__ticket_reply_times.sql index df339907..ac50e364 100644 --- a/models/reply_times/int_zendesk__ticket_reply_times.sql +++ b/models/reply_times/int_zendesk__ticket_reply_times.sql @@ -1,15 +1,19 @@ -with ticket_public_comments as ( +with commenter as ( + + select * from {{ ref('stg_zendesk__user') }} + +), ticket_public_comments as ( select * from {{ ref('int_zendesk__comments_enriched') }} - where is_public + where is_public_comment ), end_user_comments as ( select ticket_id, user_id, - valid_starting_at as end_user_comment_created_at, + valid_starting_at as comment_created_at, ticket_created_date, commenter_role, previous_internal_comment_count, @@ -25,28 +29,39 @@ with ticket_public_comments as ( end_user_comments.ticket_id, -- If the commentor was internal, a first comment, and had previous non public internal comments then we want the ticket created date to be the end user comment created date -- Otherwise we will want to end user comment created date - case when is_first_comment then end_user_comments.ticket_created_date else end_user_comments.end_user_comment_created_at end as end_user_comment_created_at, + case when is_first_comment then end_user_comments.ticket_created_date else end_user_comments.comment_created_at end as end_user_comment_created_at, end_user_comments.is_first_comment, + agent_comments.user_id as responding_agent_user_id, min(case when is_first_comment and end_user_comments.commenter_role != 'external_comment' and (end_user_comments.previous_internal_comment_count > 0) - then end_user_comments.end_user_comment_created_at + then end_user_comments.comment_created_at else agent_comments.valid_starting_at end) as agent_responded_at, - min(agent_comments.user_id) as responding_agent_user_id + rank() over (partition by end_user_comments.ticket_id, end_user_comment_created_at order by agent_responded_at asc) as rank + from end_user_comments left join ticket_public_comments as agent_comments on agent_comments.ticket_id = end_user_comments.ticket_id and agent_comments.commenter_role = 'internal_comment' - and agent_comments.valid_starting_at > end_user_comments.end_user_comment_created_at - group by 1,2,3 + and agent_comments.valid_starting_at > end_user_comments.comment_created_at + group by 1,2,3,4 + +), joined as ( + + select + reply_timestamps.*, + commenter.name as responding_agent_name, + commenter.email as responding_agent_email, + ({{ fivetran_utils.timestamp_diff( + 'end_user_comment_created_at', + 'agent_responded_at', + 'second') }} / 60.0) as reply_time_calendar_minutes + + from reply_timestamps + left join commenter on commenter.user_id = reply_timestamps.responding_agent_user_id + where rank = 1 + order by 1,2 ) - select - *, - ({{ dbt.datediff( - 'end_user_comment_created_at', - 'agent_responded_at', - 'second') }} / 60) as reply_time_calendar_minutes - from reply_timestamps - order by 1,2 \ No newline at end of file + select * from joined \ No newline at end of file diff --git a/models/zendesk.yml b/models/zendesk.yml index 8b232ac5..11221e4d 100644 --- a/models/zendesk.yml +++ b/models/zendesk.yml @@ -558,13 +558,11 @@ models: description: The time the end user created the comment - name: agent_responded_at description: Time the agent reponded to that end user comment - - name: responding_agent_user_id - description: User ID of responding agent - - name: reply_time_calendar_minutes - description: Time from end user comment created to agent response in calendar minutes - - name: reply_time_business_minutes - description: Time from end user comment created to agent response in business minutes - name: responding_agent_name description: Name of responding agent - name: responding_agent_email - description: Email of responding agent \ No newline at end of file + description: Email of responding agent + - name: reply_time_calendar_minutes + description: Time from end user comment created to agent response in calendar minutes + - name: reply_time_business_minutes + description: Time from end user comment created to agent response in business minutes \ No newline at end of file diff --git a/models/zendesk__reply_metrics.sql b/models/zendesk__reply_metrics.sql index f5092f12..7937eb2f 100644 --- a/models/zendesk__reply_metrics.sql +++ b/models/zendesk__reply_metrics.sql @@ -1,8 +1,4 @@ -with commenter as ( - - select * from {{ ref('stg_zendesk__user') }} - -), ticket_reply_times as ( +with ticket_reply_times as ( select * from {{ ref('int_zendesk__ticket_reply_times') }} @@ -19,21 +15,22 @@ with commenter as ( select ticket_reply_times.ticket_id , ticket_reply_times.end_user_comment_created_at , ticket_reply_times.agent_responded_at - , ticket_reply_times.responding_agent_user_id + , ticket_reply_times.responding_agent_name + , ticket_reply_times.responding_agent_email , ticket_reply_times.reply_time_calendar_minutes , ticket_schedules.schedule_created_at , ticket_schedules.schedule_invalidated_at , ticket_schedules.schedule_id , ({{ fivetran_utils.timestamp_diff( - "cast(" ~ dbt_date.week_start('ticket_reply_times.end_user_comment_created_at','UTC') ~ "as " ~ dbt_utils.type_timestamp() ~ ")", - "cast(ticket_reply_times.end_user_comment_created_at as " ~ dbt_utils.type_timestamp() ~ ")", + "cast(" ~ dbt_date.week_start('ticket_reply_times.end_user_comment_created_at','UTC') ~ "as " ~ dbt.type_timestamp() ~ ")", + "cast(ticket_reply_times.end_user_comment_created_at as " ~ dbt.type_timestamp() ~ ")", 'second') }} / 60.0 ) as start_time_in_minutes_from_week from ticket_reply_times join ticket_schedules on ticket_reply_times.ticket_id = ticket_schedules.ticket_id - group by 1,2,3,4,5,6,7,8 + group by 1,2,3,4,5,6,7,8,9 ), weeks as ( @@ -68,26 +65,22 @@ with commenter as ( and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule - and weekly_periods.agent_responded_at >= cast(schedule.valid_from as {{ dbt_utils.type_timestamp() }}) - and weekly_periods.agent_responded_at < cast(schedule.valid_until as {{ dbt_utils.type_timestamp() }}) + and weekly_periods.agent_responded_at >= cast(schedule.valid_from as {{ dbt.type_timestamp() }}) + and weekly_periods.agent_responded_at < cast(schedule.valid_until as {{ dbt.type_timestamp() }}) ), aggregated as ( select ticket_id , end_user_comment_created_at , agent_responded_at - , responding_agent_user_id + , responding_agent_name + , responding_agent_email , reply_time_calendar_minutes , sum(scheduled_minutes) as reply_time_business_minutes from intercepted_periods - group by 1,2,3,4,5 + group by 1,2,3,4,5,6 ) - select aggregated.* - , commenter.name as responding_agent_name - , commenter.email as responding_agent_email - from aggregated - left join commenter - on commenter.user_id = aggregated.responding_agent_user_id \ No newline at end of file + select * from aggregated \ No newline at end of file