hotfix/pass-through-all-columns

.buildkite/scripts/run_models.sh

@@ -23,4 +23,5 @@ dbt run --vars '{hubspot_marketing_enabled: true, hubspot_sales_enabled: false}'
 dbt run --vars '{hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target "$db"
 dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true}' --target "$db"
 dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: false}' --target "$db"
+dbt run --vars '{hubspot__pass_through_all_columns: true, hubspot_service_enabled: true}' --target "$db"
 dbt test --target "$db"

CHANGELOG.md

@@ -9,11 +9,14 @@ This release introduces breaking changes around how contacts are merged in order
 - Updates logic around the recently deprecated `CONTACT_MERGE_AUDIT` table.
   - The package now brings in the new `property_hs_calculated_merged_vids` field (and removes the `property_hs_` prefix) for all customers, including those on BigQuery (the field will just be `null`).
   - **Backwards-compatibility:** the package will only reference the old `CONTACT_MERGE_AUDIT` table and create `stg_hubspot__contact_merge_audit` if `hubspot_contact_merge_audit_enabled` is explicitly set to `true` in your root `dbt_project.yml` file.
-
+
+## Bug Fixes
+- The `CONTACT`, `COMPANY`, `DEAL`, and `TICKET` staging models have been updated to ensure users enabling the `hubspot__pass_through_all_columns` will have all required columns. ([PR #100](https://github.com/fivetran/dbt_hubspot_source/pull/100))
 ## Under the Hood
-[PR #98](https://github.com/fivetran/dbt_hubspot_source/pull/98) applies the following changes:
-- Updates seed data to test new merging paradigm.
-- Ensures that all timestamp fields are explicitly cast as timestamps without timezone, as recent API changes also introduced inconsistent timestamp formats.
+- Updates seed data to test new merging paradigm. ([PR #98](https://github.com/fivetran/dbt_hubspot_source/pull/98))
+- Ensures that all timestamp fields are explicitly cast as timestamps without timezone, as recent API changes also introduced inconsistent timestamp formats. ([PR #98](https://github.com/fivetran/dbt_hubspot_source/pull/98))
+- Creation of the `get_macro_columns` macro to help perform a check when enabling the `hubspot__pass_through_all_columns` to ensure the required fields are captured regardless of their existence in the source table. ([PR #100](https://github.com/fivetran/dbt_hubspot_source/pull/100))
+- Creation of the `all_passthrough_column_check` macro to help ensure that the operation to bring in all fields for the `CONTACT`, `COMPANY`, `DEAL`, and `TICKET` staging models is performed **only** if additional fields from the required are present in the source. ([PR #100](https://github.com/fivetran/dbt_hubspot_source/pull/100))
 
 See the transform package [CHANEGLOG](https://github.com/fivetran/dbt_hubspot/blob/main/CHANGELOG.md) for updates made to end models in `dbt_hubspot v0.9.0`.
 

macros/all_passthrough_column_check.sql

@@ -0,0 +1,10 @@
+{% macro all_passthrough_column_check(relation, get_columns) %}
+
+{% set available_passthrough_columns = fivetran_utils.remove_prefix_from_columns(
+                columns=adapter.get_columns_in_relation(ref(relation)), 
+                prefix='property_', exclude=get_macro_columns(get_columns))
+            %}
+
+{{ return(available_passthrough_columns|int) }}
+
+{% endmacro %}

macros/get_company_columns.sql

@@ -2,18 +2,18 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
-    {"name": "id", "datatype": dbt.type_int()},
-    {"name": "is_deleted", "datatype": "boolean"},
-    {"name": "property_name", "datatype": dbt.type_string()},
-    {"name": "property_description", "datatype": dbt.type_string()},
-    {"name": "property_createdate", "datatype": dbt.type_timestamp()},
-    {"name": "property_industry", "datatype": dbt.type_string()},
-    {"name": "property_address", "datatype": dbt.type_string()},
-    {"name": "property_address_2", "datatype": dbt.type_string()},
-    {"name": "property_city", "datatype": dbt.type_string()},
-    {"name": "property_state", "datatype": dbt.type_string()},
-    {"name": "property_country", "datatype": dbt.type_string()},
-    {"name": "property_annualrevenue", "datatype": dbt.type_int()}
+    {"name": "id", "datatype": dbt.type_int(), "alias": "company_id"},
+    {"name": "is_deleted", "datatype": "boolean", "alias": "is_company_deleted"},
+    {"name": "property_name", "datatype": dbt.type_string(), "alias": "company_name"},
+    {"name": "property_description", "datatype": dbt.type_string(), "alias": "description"},
+    {"name": "property_createdate", "datatype": dbt.type_timestamp(), "alias": "created_at"},
+    {"name": "property_industry", "datatype": dbt.type_string(), "alias": "industry"},
+    {"name": "property_address", "datatype": dbt.type_string(), "alias": "street_address"},
+    {"name": "property_address_2", "datatype": dbt.type_string(), "alias": "street_address_2"},
+    {"name": "property_city", "datatype": dbt.type_string(), "alias": "city"},
+    {"name": "property_state", "datatype": dbt.type_string(), "alias": "state"},
+    {"name": "property_country", "datatype": dbt.type_string(), "alias": "country"},
+    {"name": "property_annualrevenue", "datatype": dbt.type_int(), "alias": "company_annual_revenue"}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('hubspot__company_pass_through_columns')) }}

macros/get_contact_columns.sql

@@ -1,17 +1,17 @@
 {% macro get_contact_columns() %}
 
 {% set columns = [
-    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_deleted", "datatype": "boolean", "alias": "is_contact_deleted"},
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
-    {"name": "id", "datatype": dbt.type_int()},
-    {"name": "property_hs_calculated_merged_vids", "datatype": dbt.type_string()},
-    {"name": "property_email", "datatype": dbt.type_string()},
-    {"name": "property_company", "datatype": dbt.type_string()},
-    {"name": "property_firstname", "datatype": dbt.type_string()},
-    {"name": "property_lastname", "datatype": dbt.type_string()},
-    {"name": "property_createdate", "datatype": dbt.type_timestamp()},
-    {"name": "property_jobtitle", "datatype": dbt.type_string()},
-    {"name": "property_annualrevenue", "datatype": dbt.type_int()}
+    {"name": "id", "datatype": dbt.type_int(), "alias": "contact_id"},
+    {"name": "property_hs_calculated_merged_vids", "datatype": dbt.type_string(), "alias": "calculated_merged_vids"},
+    {"name": "property_email", "datatype": dbt.type_string(), "alias": "email"},
+    {"name": "property_company", "datatype": dbt.type_string(), "alias": "contact_company"},
+    {"name": "property_firstname", "datatype": dbt.type_string(), "alias": "first_name"},
+    {"name": "property_lastname", "datatype": dbt.type_string(), "alias": "last_name"},
+    {"name": "property_createdate", "datatype": dbt.type_timestamp(), "alias": "created_at"},
+    {"name": "property_jobtitle", "datatype": dbt.type_string(), "alias": "job_title"},
+    {"name": "property_annualrevenue", "datatype": dbt.type_int(), "alias": "company_annual_revenue"}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('hubspot__contact_pass_through_columns')) }}

macros/get_deal_columns.sql

@@ -5,14 +5,14 @@
     {"name": "deal_id", "datatype": dbt.type_int()},
     {"name": "deal_pipeline_id", "datatype": dbt.type_string()},
     {"name": "deal_pipeline_stage_id", "datatype": dbt.type_string()},
-    {"name": "is_deleted", "datatype": "boolean"},
+    {"name": "is_deleted", "datatype": "boolean", "alias": "is_deal_deleted"},
     {"name": "owner_id", "datatype": dbt.type_int()},
     {"name": "portal_id", "datatype": dbt.type_int()},
-    {"name": "property_dealname", "datatype": dbt.type_string()},
-    {"name": "property_description", "datatype": dbt.type_string()},
-    {"name": "property_amount", "datatype": dbt.type_int()},
-    {"name": "property_closedate", "datatype": dbt.type_timestamp()},
-    {"name": "property_createdate", "datatype": dbt.type_timestamp()}
+    {"name": "property_dealname", "datatype": dbt.type_string(), "alias": "deal_name"},
+    {"name": "property_description", "datatype": dbt.type_string(), "alias": "description"},
+    {"name": "property_amount", "datatype": dbt.type_int(), "alias": "amount"},
+    {"name": "property_closedate", "datatype": dbt.type_timestamp(), "alias": "closed_at"},
+    {"name": "property_createdate", "datatype": dbt.type_timestamp(), "alias": "created_at"}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('hubspot__deal_pass_through_columns')) }}

macros/get_macro_columns.sql

@@ -0,0 +1,9 @@
+{%- macro get_macro_columns(get_column_macro) -%}
+
+    {%- set macro_column_names = [] -%}
+    {%- for col in get_column_macro -%}
+        {%- set macro_column_names = macro_column_names.append(col.name) -%}
+    {%- endfor -%}
+
+{{ return(macro_column_names) }}
+{%- endmacro -%}

macros/get_ticket_columns.sql

@@ -2,18 +2,18 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
-    {"name": "id", "datatype": dbt.type_int()},
-    {"name": "is_deleted", "datatype": "boolean"},
-    {"name": "property_closed_date", "datatype": dbt.type_timestamp()},
-    {"name": "property_createdate", "datatype": dbt.type_timestamp()},
-    {"name": "property_first_agent_reply_date", "datatype": dbt.type_timestamp()},
-    {"name": "property_hs_pipeline", "datatype": dbt.type_string()},
-    {"name": "property_hs_pipeline_stage", "datatype": dbt.type_string()},
-    {"name": "property_hs_ticket_category", "datatype": dbt.type_string()},
-    {"name": "property_hs_ticket_priority", "datatype": dbt.type_string()},
-    {"name": "property_hubspot_owner_id", "datatype": dbt.type_int()},
-    {"name": "property_subject", "datatype": dbt.type_string()},
-    {"name": "property_content", "datatype": dbt.type_string()}  
+    {"name": "id", "datatype": dbt.type_int(), "alias": "ticket_id"},
+    {"name": "is_deleted", "datatype": "boolean", "alias": "is_ticket_deleted"},
+    {"name": "property_closed_date", "datatype": dbt.type_timestamp(), "alias": "closed_at"},
+    {"name": "property_createdate", "datatype": dbt.type_timestamp(), "alias": "created_at"},
+    {"name": "property_first_agent_reply_date", "datatype": dbt.type_timestamp(), "alias": "first_agent_reply_at"},
+    {"name": "property_hs_pipeline", "datatype": dbt.type_string(), "alias": "ticket_pipeline_id"},
+    {"name": "property_hs_pipeline_stage", "datatype": dbt.type_string(), "alias": "ticket_pipeline_stage_id"},
+    {"name": "property_hs_ticket_category", "datatype": dbt.type_string(), "alias": "ticket_category"},
+    {"name": "property_hs_ticket_priority", "datatype": dbt.type_string(), "alias": "ticket_priority"},
+    {"name": "property_hubspot_owner_id", "datatype": dbt.type_int(), "alias": "owner_id"},
+    {"name": "property_subject", "datatype": dbt.type_string(), "alias": "ticket_subject"},
+    {"name": "property_content", "datatype": dbt.type_string(), "alias": "ticket_content"}  
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('hubspot__ticket_pass_through_columns')) }}

models/stg_hubspot__company.sql

@@ -19,33 +19,40 @@ with base as (
 ), fields as (
 
     select
-        id as company_id,
-        is_deleted as is_company_deleted,
 
 {% if var('hubspot__pass_through_all_columns', false) %}
-        -- just pass everything through
-        {{ 
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_hubspot__company_tmp')),
+                staging_columns=get_company_columns()
+            )
+        }}
+        {% if all_passthrough_column_check('stg_hubspot__company_tmp',get_company_columns()) > 0 %}
+        -- just pass everything through if extra columns are present, but ensure required columns are present.
+        ,{{ 
             fivetran_utils.remove_prefix_from_columns(
                 columns=adapter.get_columns_in_relation(ref('stg_hubspot__company_tmp')), 
-                prefix='property_', exclude=['id', 'is_deleted']
-            ) 
+                prefix='property_', exclude=get_macro_columns(get_company_columns()))
         }}
-
+        {% endif %}
     from base
 
 {% else %}
         -- just default columns + explicitly configured passthrough columns
+        -- a few columns below are aliased within the macros/get_company_columns.sql macro
+        company_id,
+        is_company_deleted,
         cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced,
-        property_name as company_name,
-        property_description as description,
-        property_createdate as created_at,
-        property_industry as industry,
-        property_address as street_address,
-        property_address_2 as street_address_2,
-        property_city as city,
-        property_state as state,
-        property_country as country,
-        property_annualrevenue as company_annual_revenue
+        company_name,
+        description,
+        created_at,
+        industry,
+        street_address,
+        street_address_2,
+        city,
+        state,
+        country,
+        company_annual_revenue
 
         --The below macro adds the fields defined within your hubspot__ticket_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('hubspot__company_pass_through_columns') }}

models/stg_hubspot__contact.sql

@@ -19,28 +19,37 @@ with base as (
 ), fields as (
 
     select
-        id as contact_id,
-        _fivetran_deleted as is_contact_deleted,
 
 {% if var('hubspot__pass_through_all_columns', false) %}
-        -- just pass everything through
-        {{ 
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_hubspot__contact_tmp')),
+                staging_columns=get_contact_columns()
+            )
+        }}
+        {% if all_passthrough_column_check('stg_hubspot__contact_tmp',get_contact_columns()) > 0 %}
+        -- just pass everything through if extra columns are present, but ensure required columns are present.
+        ,{{ 
             fivetran_utils.remove_prefix_from_columns(
                 columns=adapter.get_columns_in_relation(ref('stg_hubspot__contact_tmp')), 
-                prefix='property_', exclude=['id', 'property_contact_id','_fivetran_deleted','property_hs_calculated_merged_vids']) 
+                prefix='property_', exclude=get_macro_columns(get_contact_columns())) 
         }}
+        {% endif %}
     from base
 
 {% else %}
-        -- just default columns + explicitly configured passthrough columns
-        property_hs_calculated_merged_vids as calculated_merged_vids, -- will be null for BigQuery users until v3 api is rolled out to them
-        property_email as email,
-        property_company as contact_company,
-        property_firstname as first_name,
-        property_lastname as last_name,
-        cast(property_createdate as {{ dbt.type_timestamp() }}) as created_at,
-        property_jobtitle as job_title,
-        property_annualrevenue as company_annual_revenue,
+        -- just default columns + explicitly configured passthrough columns.
+        -- a few columns below are aliased within the macros/get_contact_columns.sql macro
+        contact_id,
+        is_contact_deleted,
+        calculated_merged_vids, -- will be null for BigQuery users until v3 api is rolled out to them
+        email,
+        contact_company,
+        first_name,
+        last_name,
+        cast(created_at as {{ dbt.type_timestamp() }}) as created_at,
+        job_title,
+        company_annual_revenue,
         cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced
 
         --The below macro adds the fields defined within your hubspot__contact_pass_through_columns variable into the staging model

models/stg_hubspot__deal.sql

@@ -19,31 +19,39 @@ with base as (
 ), fields as (
 
     select
-        property_dealname as deal_name,
-        cast(property_closedate as {{ dbt.type_timestamp() }}) as closed_at,
-        cast(property_createdate as {{ dbt.type_timestamp() }}) as created_at,
-        is_deleted as is_deal_deleted,
 
 {% if var('hubspot__pass_through_all_columns', false) %}
-        -- just pass everything else through
-        {{ 
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_hubspot__deal_tmp')),
+                staging_columns=get_deal_columns()
+            )
+        }}
+        {% if all_passthrough_column_check('stg_hubspot__deal_tmp',get_deal_columns()) > 0 %}
+        -- just pass everything through if extra columns are present, but ensure required columns are present.
+        ,{{ 
             fivetran_utils.remove_prefix_from_columns(
                 columns=adapter.get_columns_in_relation(ref('stg_hubspot__deal_tmp')), 
-                prefix='property_',
-                exclude=['property_dealname','property_closedate','property_createdate', 'is_deleted']) 
+                prefix='property_',exclude=get_macro_columns(get_deal_columns()))
         }}
+        {% endif %}
     from base
 
 {% else %}
         -- just default columns + explicitly configured passthrough columns
+        -- a few columns below are aliased within the macros/get_deal_columns.sql macro
+        deal_name,
+        cast(closed_at as {{ dbt.type_timestamp() }}) as closed_at,
+        cast(created_at as {{ dbt.type_timestamp() }}) as created_at,
+        is_deal_deleted,
         cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced,
         deal_id,
         cast(deal_pipeline_id as {{ dbt.type_string() }}) as deal_pipeline_id,
         cast(deal_pipeline_stage_id as {{ dbt.type_string() }}) as deal_pipeline_stage_id,
         owner_id,
         portal_id,
-        property_description as description,
-        property_amount as amount
+        description,
+        amount
 
         --The below macro adds the fields defined within your hubspot__deal_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('hubspot__deal_pass_through_columns') }}