From b2f74a456d6353343254a5aae94599effafb6a75 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Thu, 2 Sep 2021 16:09:19 -0400 Subject: [PATCH 01/13] try week granularity --- models/ticket_history/int_zendesk__field_calendar_spine.sql | 2 +- models/ticket_history/int_zendesk__field_history_pivot.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index bc398035..54735d63 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -1,7 +1,7 @@ {{ config( materialized='incremental', - partition_by = {'field': 'date_day', 'data_type': 'date'}, + partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'week'}, unique_key='ticket_day_id' ) }} diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index 6df89cea..e5aa6681 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -3,7 +3,7 @@ {{ config( materialized='incremental', - partition_by = {'field': 'date_day', 'data_type': 'date'}, + partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'week'}, unique_key='ticket_day_id' ) }} From ba1072aea017c0c0678c535064601df85ad30471 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Wed, 8 Sep 2021 15:23:06 -0700 Subject: [PATCH 02/13] month granularity --- models/ticket_history/int_zendesk__field_calendar_spine.sql | 2 +- models/ticket_history/int_zendesk__field_history_pivot.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index 54735d63..8e9b1d43 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -1,7 +1,7 @@ {{ config( materialized='incremental', - partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'week'}, + partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'month'}, unique_key='ticket_day_id' ) }} diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index e5aa6681..ca83c04a 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -3,7 +3,7 @@ {{ config( materialized='incremental', - partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'week'}, + partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'month'}, unique_key='ticket_day_id' ) }} From bb27f235f7d0c604108d76a6bc34d77725ab1c19 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Tue, 14 Sep 2021 12:01:54 -0700 Subject: [PATCH 03/13] add cluster_by --- models/ticket_history/int_zendesk__field_calendar_spine.sql | 3 ++- models/ticket_history/int_zendesk__field_history_pivot.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index 8e9b1d43..8c501d20 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -2,7 +2,8 @@ config( materialized='incremental', partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'month'}, - unique_key='ticket_day_id' + unique_key='ticket_day_id', + cluster_by='ticket_id' ) }} diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index ca83c04a..d40fc971 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -4,7 +4,8 @@ config( materialized='incremental', partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'month'}, - unique_key='ticket_day_id' + unique_key='ticket_day_id', + cluster_by='ticket_id' ) }} From c07a879f75fc814fe00336c1ebb14327528d3101 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Thu, 16 Sep 2021 12:03:23 -0700 Subject: [PATCH 04/13] revert monthly granularity + clusters --- models/ticket_history/int_zendesk__field_calendar_spine.sql | 5 ++--- models/ticket_history/int_zendesk__field_history_pivot.sql | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index 8c501d20..bc398035 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -1,9 +1,8 @@ {{ config( materialized='incremental', - partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'month'}, - unique_key='ticket_day_id', - cluster_by='ticket_id' + partition_by = {'field': 'date_day', 'data_type': 'date'}, + unique_key='ticket_day_id' ) }} diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index d40fc971..6df89cea 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -3,9 +3,8 @@ {{ config( materialized='incremental', - partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': 'month'}, - unique_key='ticket_day_id', - cluster_by='ticket_id' + partition_by = {'field': 'date_day', 'data_type': 'date'}, + unique_key='ticket_day_id' ) }} From f4bb026c1481cdf779af0c36b7cc51ab88b7283e Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Thu, 16 Sep 2021 12:45:23 -0700 Subject: [PATCH 05/13] put cap on ticket timeline --- .../ticket_history/int_zendesk__field_calendar_spine.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index bc398035..1a9b7375 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -16,7 +16,10 @@ with calendar as ( ), ticket as ( - select * + select + *, + -- closed tickets cannot be re-opened or updated, and solved tickets are automatically closed after a pre-defined number of days configured in your Zendesk settings + cast( {{ dbt_utils.date_trunc('day', "case when status != 'closed' then " ~ dbt_utils.current_timestamp() ~ " else updated_at end") }} as date) as open_until from {{ var('ticket') }} ), joined as ( @@ -27,6 +30,8 @@ with calendar as ( from calendar inner join ticket on calendar.date_day >= cast(ticket.created_at as date) + -- use this variable to extend the ticket's history past its close date (for reporting/data viz purposes :-) + and {{ dbt_utils.dateadd('month', var('ticket_history_extension', 0), 'ticket.open_until') }} >= calendar.date_day ), surrogate_key as ( From d0a35de58e2c1b1afc66ac74a31bfa2330aa16dc Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Thu, 16 Sep 2021 12:52:02 -0700 Subject: [PATCH 06/13] re-add clusters --- models/ticket_history/int_zendesk__field_calendar_spine.sql | 3 ++- models/ticket_history/int_zendesk__field_history_pivot.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index 1a9b7375..d0a9e274 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -2,7 +2,8 @@ config( materialized='incremental', partition_by = {'field': 'date_day', 'data_type': 'date'}, - unique_key='ticket_day_id' + unique_key='ticket_day_id', + cluster_by='ticket_id' ) }} diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index 6df89cea..b4720227 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -4,7 +4,8 @@ config( materialized='incremental', partition_by = {'field': 'date_day', 'data_type': 'date'}, - unique_key='ticket_day_id' + unique_key='ticket_day_id', + cluster_by='ticket_id' ) }} From c5784116a2a655d3a0290636abb962dac5d13d65 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Thu, 16 Sep 2021 12:54:40 -0700 Subject: [PATCH 07/13] circle config --- integration_tests/dbt_project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index d2cee969..a9ab1260 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -71,6 +71,7 @@ seeds: id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" created_at: timestamp due_at: timestamp + updated_at: timestamp assignee_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" brand_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" external_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" From c73ff46f636c14ccb34492b66baa5eb89f41e11d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Thu, 16 Sep 2021 14:55:48 -0700 Subject: [PATCH 08/13] adapted jira code + remove clusters --- models/ticket_history/int_zendesk__field_calendar_spine.sql | 5 ++--- models/ticket_history/int_zendesk__field_history_pivot.sql | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index d0a9e274..b56d90d7 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -1,9 +1,8 @@ {{ config( materialized='incremental', - partition_by = {'field': 'date_day', 'data_type': 'date'}, - unique_key='ticket_day_id', - cluster_by='ticket_id' + partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': var('partition_by_granularity', 'month')}, + unique_key='ticket_day_id' ) }} diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index b4720227..8c9ec59d 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -3,9 +3,8 @@ {{ config( materialized='incremental', - partition_by = {'field': 'date_day', 'data_type': 'date'}, - unique_key='ticket_day_id', - cluster_by='ticket_id' + partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': var('partition_by_granularity', 'month')}, + unique_key='ticket_day_id' ) }} From 17eb9d82751edf4680eda2920548b1e9546934ff Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Fri, 24 Sep 2021 13:48:58 -0700 Subject: [PATCH 09/13] switch to date range variable --- README.md | 2 ++ dbt_project.yml | 2 +- models/ticket_history/int_zendesk__field_calendar_spine.sql | 2 +- models/ticket_history/int_zendesk__field_history_pivot.sql | 2 +- models/utils/int_zendesk__calendar_spine.sql | 3 +++ 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2ae09449..cffcb818 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,8 @@ vars: ``` *Note: This package only integrates the above variables. If you'd like to disable other models, please create an [issue](https://github.com/fivetran/dbt_zendesk/issues) specifying which ones.* +### Limiting Ticket Field History Date Range +[] todo after approving the code! ## Database support This package is compatible with BigQuery, Snowflake, Redshift and Postgres. diff --git a/dbt_project.yml b/dbt_project.yml index 166fdb49..7423475f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'zendesk' -version: '0.6.0' +version: '0.6.1' config-version: 2 require-dbt-version: [">=0.20.0"] on-run-start: '{{ fivetran_utils.empty_variable_warning("ticket_field_history_columns", "zendesk_ticket_field_history") }}' diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index b56d90d7..1a9b7375 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -1,7 +1,7 @@ {{ config( materialized='incremental', - partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': var('partition_by_granularity', 'month')}, + partition_by = {'field': 'date_day', 'data_type': 'date'}, unique_key='ticket_day_id' ) }} diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index 8c9ec59d..6df89cea 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -3,7 +3,7 @@ {{ config( materialized='incremental', - partition_by = {'field': 'date_day', 'data_type': 'date', 'granularity': var('partition_by_granularity', 'month')}, + partition_by = {'field': 'date_day', 'data_type': 'date'}, unique_key='ticket_day_id' ) }} diff --git a/models/utils/int_zendesk__calendar_spine.sql b/models/utils/int_zendesk__calendar_spine.sql index 618f82a8..f1301a93 100644 --- a/models/utils/int_zendesk__calendar_spine.sql +++ b/models/utils/int_zendesk__calendar_spine.sql @@ -33,6 +33,9 @@ with spine as ( select cast(date_day as date) as date_day from spine + -- by default take all the data + where date_day >= {{ dbt_utils.dateadd('year', - var('ticket_field_history_timeframe', 50), dbt_utils.current_timestamp() ) }} + ) select * From 183c3b16eb9215548e69115da46f170c85c07d47 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Mon, 4 Oct 2021 13:51:15 -0700 Subject: [PATCH 10/13] standardize variable names --- dbt_project.yml | 2 ++ models/ticket_history/int_zendesk__field_calendar_spine.sql | 2 +- models/utils/int_zendesk__calendar_spine.sql | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 7423475f..85bf0918 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -48,3 +48,5 @@ vars: using_ticket_form_history: true using_organization_tags: true + ticket_field_history_extension_months: 0 # how long to extend a ticket's field history past its closure date + ticket_field_history_timeframe_years: 50 # how far back to pull tickets' field histories. default is everything \ No newline at end of file diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index 1a9b7375..46adf7f2 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -31,7 +31,7 @@ with calendar as ( inner join ticket on calendar.date_day >= cast(ticket.created_at as date) -- use this variable to extend the ticket's history past its close date (for reporting/data viz purposes :-) - and {{ dbt_utils.dateadd('month', var('ticket_history_extension', 0), 'ticket.open_until') }} >= calendar.date_day + and {{ dbt_utils.dateadd('month', var('ticket_field_history_extension_months', 0), 'ticket.open_until') }} >= calendar.date_day ), surrogate_key as ( diff --git a/models/utils/int_zendesk__calendar_spine.sql b/models/utils/int_zendesk__calendar_spine.sql index f1301a93..7c290321 100644 --- a/models/utils/int_zendesk__calendar_spine.sql +++ b/models/utils/int_zendesk__calendar_spine.sql @@ -34,7 +34,7 @@ with spine as ( from spine -- by default take all the data - where date_day >= {{ dbt_utils.dateadd('year', - var('ticket_field_history_timeframe', 50), dbt_utils.current_timestamp() ) }} + where date_day >= {{ dbt_utils.dateadd('year', - var('ticket_field_history_timeframe_years', 50), dbt_utils.current_timestamp() ) }} ) From a2162a9bd634d5b35b4619752658df0dece07284 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Mon, 4 Oct 2021 14:22:39 -0700 Subject: [PATCH 11/13] add to readme --- README.md | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cffcb818..dc047aef 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ vars: zendesk_schema: your_schema_name ``` +### Tracking Ticket Field History Columns The `zendesk__ticket_field_history` model generates historical data for the columns specified by the `ticket_field_history_columns` variable. By default, the columns tracked are `status`, `priority`, and `assignee_id`. If you would like to change these columns, add the following configuration to your `dbt_project.yml` file. Additionally, the `zendesk__ticket_field_history` model allows for tracking the specified fields updater information through the use of the `zendesk_ticket_field_history_updater_columns` variable. The values passed through this variable limited to the values shown within the config below. By default, the variable is empty and updater information is not tracked. If you would like to track field history updater information, add any of the below specified values to your `dbt_project.yml` file. After adding the columns to your `dbt_project.yml` file, run the `dbt run --full-refresh` command to fully refresh any existing models. ```yml @@ -88,7 +89,7 @@ models: ``` -### Disabling models +### Disabling Models This package takes into consideration that not every Zendesk account utilizes the `schedule`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable: @@ -107,8 +108,34 @@ vars: ``` *Note: This package only integrates the above variables. If you'd like to disable other models, please create an [issue](https://github.com/fivetran/dbt_zendesk/issues) specifying which ones.* -### Limiting Ticket Field History Date Range -[] todo after approving the code! +### Extending and Limiting the Ticket Field History + +This package will create a row in `zendesk__ticket_field_history` for each day that a ticket is open, starting at its creation date. A Zendesk ticket cannot be altered after being closed, so its field values will not change after this date. However, you may want to extend a ticket's history past its closure date for easier reporting and visualizing. To do so, add the following configuration to your `dbt_project.yml` file: + +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + zendesk: + ticket_field_history_extension_months: integer_number_of_months # default = 0 +``` + +Conversely, you may want to only track the past X years of ticket field history. This could be for cost reasons, or because you have a BigQuery destination and have over 4,000 days (10-11 years) of data, leading to a `too many partitions` error in the package's incremental models. To limit the ticket field history to the most recent X years, add the following configuration to your `dbt_project.yml` file: + +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + zendesk: + ticket_field_history_timeframe_years: integer_number_of_years # default = 50 (everything) +``` + ## Database support This package is compatible with BigQuery, Snowflake, Redshift and Postgres. From 6f1e30b6eac62175d0de07f7fed3b0e10cb3a125 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Tue, 5 Oct 2021 13:46:07 -0700 Subject: [PATCH 12/13] move where clause --- models/utils/int_zendesk__calendar_spine.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/utils/int_zendesk__calendar_spine.sql b/models/utils/int_zendesk__calendar_spine.sql index 7c290321..2926e12b 100644 --- a/models/utils/int_zendesk__calendar_spine.sql +++ b/models/utils/int_zendesk__calendar_spine.sql @@ -5,7 +5,10 @@ with spine as ( {% if execute %} {% set first_date_query %} select min( created_at ) as min_date from {{ ref('stg_zendesk__ticket') }} + -- by default take all the data + where cast(created_at as date) >= {{ dbt_utils.dateadd('year', - var('ticket_field_history_timeframe_years', 50), dbt_utils.current_timestamp() ) }} {% endset %} + {% set first_date = run_query(first_date_query).columns[0][0]|string %} {% if target.type == 'postgres' %} @@ -33,9 +36,6 @@ with spine as ( select cast(date_day as date) as date_day from spine - -- by default take all the data - where date_day >= {{ dbt_utils.dateadd('year', - var('ticket_field_history_timeframe_years', 50), dbt_utils.current_timestamp() ) }} - ) select * From c2fabea2c4facb981e5cb219023534494cf97381 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez Date: Tue, 5 Oct 2021 13:48:22 -0700 Subject: [PATCH 13/13] update package versioin --- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 85bf0918..f85c63a9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'zendesk' -version: '0.6.1' +version: '0.7.0' config-version: 2 require-dbt-version: [">=0.20.0"] on-run-start: '{{ fivetran_utils.empty_variable_warning("ticket_field_history_columns", "zendesk_ticket_field_history") }}' diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index a9ab1260..46b44198 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'zendesk_integration_tests' -version: '0.6.0' +version: '0.7.0' profile: 'integration_tests'