diff --git a/README.md b/README.md index 2ae09449..dc047aef 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ vars: zendesk_schema: your_schema_name ``` +### Tracking Ticket Field History Columns The `zendesk__ticket_field_history` model generates historical data for the columns specified by the `ticket_field_history_columns` variable. By default, the columns tracked are `status`, `priority`, and `assignee_id`. If you would like to change these columns, add the following configuration to your `dbt_project.yml` file. Additionally, the `zendesk__ticket_field_history` model allows for tracking the specified fields updater information through the use of the `zendesk_ticket_field_history_updater_columns` variable. The values passed through this variable limited to the values shown within the config below. By default, the variable is empty and updater information is not tracked. If you would like to track field history updater information, add any of the below specified values to your `dbt_project.yml` file. After adding the columns to your `dbt_project.yml` file, run the `dbt run --full-refresh` command to fully refresh any existing models. ```yml @@ -88,7 +89,7 @@ models: ``` -### Disabling models +### Disabling Models This package takes into consideration that not every Zendesk account utilizes the `schedule`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable: @@ -107,6 +108,34 @@ vars: ``` *Note: This package only integrates the above variables. If you'd like to disable other models, please create an [issue](https://github.com/fivetran/dbt_zendesk/issues) specifying which ones.* +### Extending and Limiting the Ticket Field History + +This package will create a row in `zendesk__ticket_field_history` for each day that a ticket is open, starting at its creation date. A Zendesk ticket cannot be altered after being closed, so its field values will not change after this date. However, you may want to extend a ticket's history past its closure date for easier reporting and visualizing. To do so, add the following configuration to your `dbt_project.yml` file: + +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + zendesk: + ticket_field_history_extension_months: integer_number_of_months # default = 0 +``` + +Conversely, you may want to only track the past X years of ticket field history. This could be for cost reasons, or because you have a BigQuery destination and have over 4,000 days (10-11 years) of data, leading to a `too many partitions` error in the package's incremental models. To limit the ticket field history to the most recent X years, add the following configuration to your `dbt_project.yml` file: + +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + zendesk: + ticket_field_history_timeframe_years: integer_number_of_years # default = 50 (everything) +``` + ## Database support This package is compatible with BigQuery, Snowflake, Redshift and Postgres. diff --git a/dbt_project.yml b/dbt_project.yml index 7423475f..f85c63a9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'zendesk' -version: '0.6.1' +version: '0.7.0' config-version: 2 require-dbt-version: [">=0.20.0"] on-run-start: '{{ fivetran_utils.empty_variable_warning("ticket_field_history_columns", "zendesk_ticket_field_history") }}' @@ -48,3 +48,5 @@ vars: using_ticket_form_history: true using_organization_tags: true + ticket_field_history_extension_months: 0 # how long to extend a ticket's field history past its closure date + ticket_field_history_timeframe_years: 50 # how far back to pull tickets' field histories. default is everything \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2deea1e6..46b44198 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'zendesk_integration_tests' -version: '0.6.1' +version: '0.7.0' profile: 'integration_tests' @@ -71,6 +71,7 @@ seeds: id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" created_at: timestamp due_at: timestamp + updated_at: timestamp assignee_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" brand_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" external_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index bc398035..46adf7f2 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -16,7 +16,10 @@ with calendar as ( ), ticket as ( - select * + select + *, + -- closed tickets cannot be re-opened or updated, and solved tickets are automatically closed after a pre-defined number of days configured in your Zendesk settings + cast( {{ dbt_utils.date_trunc('day', "case when status != 'closed' then " ~ dbt_utils.current_timestamp() ~ " else updated_at end") }} as date) as open_until from {{ var('ticket') }} ), joined as ( @@ -27,6 +30,8 @@ with calendar as ( from calendar inner join ticket on calendar.date_day >= cast(ticket.created_at as date) + -- use this variable to extend the ticket's history past its close date (for reporting/data viz purposes :-) + and {{ dbt_utils.dateadd('month', var('ticket_field_history_extension_months', 0), 'ticket.open_until') }} >= calendar.date_day ), surrogate_key as ( diff --git a/models/utils/int_zendesk__calendar_spine.sql b/models/utils/int_zendesk__calendar_spine.sql index 618f82a8..2926e12b 100644 --- a/models/utils/int_zendesk__calendar_spine.sql +++ b/models/utils/int_zendesk__calendar_spine.sql @@ -5,7 +5,10 @@ with spine as ( {% if execute %} {% set first_date_query %} select min( created_at ) as min_date from {{ ref('stg_zendesk__ticket') }} + -- by default take all the data + where cast(created_at as date) >= {{ dbt_utils.dateadd('year', - var('ticket_field_history_timeframe_years', 50), dbt_utils.current_timestamp() ) }} {% endset %} + {% set first_date = run_query(first_date_query).columns[0][0]|string %} {% if target.type == 'postgres' %}