Skip to content

Commit

Permalink
Add snowflake lake loader support (snowplow#59)
Browse files Browse the repository at this point in the history 
* Make changes for snowflake lake loader support

* hard-cast fields

* reset project file

* lake loader isn't on by default

* Add note about lake loader workaround

* fix falses
  • Loading branch information
@rlh1994 @jedichien
rlh1994 authored and jedichien committed Jul 2, 2024
1 parent c821e23 commit 6f8d381
Show file tree
Hide file tree
Showing 16 changed files with 315 additions and 128 deletions.
1 change: 1 addition & 0 deletions dbt_project.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ vars:

# Snowflake:
snowplow__query_tag: snowplow_dbt
snowplow__snowflake_lakeloader: false

# Databricks:
# Depending on the use case it should either be the catalog (for Unity Catalog users from databricks connector 1.1.1 onwards) or the same value as your snowplow__atomic_schema (unless changed it should be 'atomic')
Expand Down
31 changes: 22 additions & 9 deletions macros/field_extractions/get_app_error_event_fields.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,15 +84,28 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0

{% macro snowflake__get_app_error_event_fields() %}
{% if var('snowplow__enable_app_errors', false) %}
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:message::VARCHAR() AS app_error__message
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:programmingLanguage::VARCHAR() AS app_error__programming_language
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:className::VARCHAR() AS app_error__class_name
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:exceptionName::VARCHAR() AS app_error__exception_name
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:isFatal::BOOLEAN AS app_error__is_fatal
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:lineNumber::INT AS app_error__line_number
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:stackTrace::VARCHAR() AS app_error__stack_trace
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadId::INT AS app_error__thread_id
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadName::VARCHAR() AS app_error__thread_name
{% if var('snowplow__snowflake_lakeloader', false) %}
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:message::VARCHAR() AS app_error__message
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:programming_language::VARCHAR() AS app_error__programming_language
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:class_name::VARCHAR() AS app_error__class_name
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:exception_name::VARCHAR() AS app_error__exception_name
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:is_fatal::BOOLEAN AS app_error__is_fatal
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:line_number::INT AS app_error__line_number
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:stack_trace::VARCHAR() AS app_error__stack_trace
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:thread_id::INT AS app_error__thread_id
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:thread_name::VARCHAR() AS app_error__thread_name
{% else %}
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:message::VARCHAR() AS app_error__message
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:programmingLanguage::VARCHAR() AS app_error__programming_language
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:className::VARCHAR() AS app_error__class_name
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:exceptionName::VARCHAR() AS app_error__exception_name
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:isFatal::BOOLEAN AS app_error__is_fatal
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:lineNumber::INT AS app_error__line_number
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:stackTrace::VARCHAR() AS app_error__stack_trace
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadId::INT AS app_error__thread_id
, unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadName::VARCHAR() AS app_error__thread_name
{% endif %}

{% else %}
, cast(null as {{ dbt.type_string() }}) as app_error__message
, cast(null as {{ dbt.type_string() }}) as app_error__programming_language
Expand Down
42 changes: 29 additions & 13 deletions macros/field_extractions/get_browser_context_fields.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,19 +105,35 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0

{% macro snowflake__get_browser_context_fields() %}
{% if var('snowplow__enable_browser_context', false) %}
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:viewport::varchar AS browser__viewport
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentSize::varchar AS browser__document_size
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:resolution::varchar AS browser__resolution
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:colorDepth::int AS browser__color_depth
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:devicePixelRatio::float AS browser__device_pixel_ratio
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:cookiesEnabled::boolean AS browser__cookies_enabled
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:online::boolean AS browser__online
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:browserLanguage::varchar AS browser__browser_language
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentLanguage::varchar AS browser__document_language
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:webdriver::boolean AS browser__webdriver
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:deviceMemory::int AS browser__device_memory
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:hardwareConcurrency::int AS browser__hardware_concurrency
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:tabId::varchar AS browser__tab_id
{% if var('snowplow__snowflake_lakeloader', false) %}
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:viewport::varchar AS browser__viewport
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:document_size::varchar AS browser__document_size
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:resolution::varchar AS browser__resolution
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:color_depth::int AS browser__color_depth
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:device_pixel_ratio::float AS browser__device_pixel_ratio
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:cookies_enabled::boolean AS browser__cookies_enabled
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:online::boolean AS browser__online
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:browser_language::varchar AS browser__browser_language
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:document_language::varchar AS browser__document_language
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:webdriver::boolean AS browser__webdriver
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:device_memory::int AS browser__device_memory
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:hardware_concurrency::int AS browser__hardware_concurrency
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:tab_id::varchar AS browser__tab_id
{% else %}
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:viewport::varchar AS browser__viewport
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentSize::varchar AS browser__document_size
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:resolution::varchar AS browser__resolution
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:colorDepth::int AS browser__color_depth
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:devicePixelRatio::float AS browser__device_pixel_ratio
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:cookiesEnabled::boolean AS browser__cookies_enabled
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:online::boolean AS browser__online
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:browserLanguage::varchar AS browser__browser_language
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentLanguage::varchar AS browser__document_language
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:webdriver::boolean AS browser__webdriver
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:deviceMemory::int AS browser__device_memory
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:hardwareConcurrency::int AS browser__hardware_concurrency
, contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:tabId::varchar AS browser__tab_id
{% endif %}
{% else %}
, cast(null as {{ dbt.type_string() }}) as browser__viewport
, cast(null as {{ dbt.type_string() }}) as browser__document_size
Expand Down
6 changes: 5 additions & 1 deletion macros/field_extractions/get_cmp_visible_event_fields.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,11 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0

{% macro snowflake__get_cmp_visible_event_fields() %}
{% if var('snowplow__enable_consent', false) %}
, unstruct_event_com_snowplowanalytics_snowplow_cmp_visible_1:elapsedTime::float as cmp__elapsed_time
{% if var('snowplow__snowflake_lakeloader', false) %}
, unstruct_event_com_snowplowanalytics_snowplow_cmp_visible_1:elapsed_time::float as cmp__elapsed_time
{% else %}
, unstruct_event_com_snowplowanalytics_snowplow_cmp_visible_1:elapsedTime::float as cmp__elapsed_time
{% endif %}
{% else %}
, cast(null as {{ dbt.type_float() }}) as cmp__elapsed_time
{% endif %}
Expand Down
43 changes: 26 additions & 17 deletions macros/field_extractions/get_consent_event_fields.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,23 +74,32 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
{% endmacro %}

{% macro snowflake__get_consent_event_fields() %}
{% if var('snowplow__enable_consent', false) %}

, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:eventType::varchar as consent__event_type
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:basisForProcessing::varchar as consent__basis_for_processing
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentUrl::varchar as consent__consent_url
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentVersion::varchar as consent__consent_version
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentScopes::array as consent__consent_scopes
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:domainsApplied::array as consent__domains_applied
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:gdprApplies::boolean as consent__gdpr_applies

{% if var('snowplow__enable_consent', false) %}
{% if var('snowplow__snowflake_lakeloader', false) %}
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:event_type::varchar as consent__event_type
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:basis_for_processing::varchar as consent__basis_for_processing
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consent_url::varchar as consent__consent_url
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consent_version::varchar as consent__consent_version
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consent_scopes::array as consent__consent_scopes
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:domains_applied::array as consent__domains_applied
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:gdpr_applies::boolean as consent__gdpr_applies
{% else %}
, cast(null as {{ dbt.type_string() }}) as consent__event_type
, cast(null as {{ dbt.type_string() }}) as consent__basis_for_processing
, cast(null as {{ dbt.type_string() }}) as consent__consent_url
, cast(null as {{ dbt.type_string() }}) as consent__consent_version
, cast(null as array) as consent__consent_scopes
, cast(null as array) as consent__domains_applied
, cast(null as {{ dbt.type_boolean() }}) as consent__gdpr_applies
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:eventType::varchar as consent__event_type
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:basisForProcessing::varchar as consent__basis_for_processing
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentUrl::varchar as consent__consent_url
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentVersion::varchar as consent__consent_version
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentScopes::array as consent__consent_scopes
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:domainsApplied::array as consent__domains_applied
, unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:gdprApplies::boolean as consent__gdpr_applies
{% endif %}

{% else %}
, cast(null as {{ dbt.type_string() }}) as consent__event_type
, cast(null as {{ dbt.type_string() }}) as consent__basis_for_processing
, cast(null as {{ dbt.type_string() }}) as consent__consent_url
, cast(null as {{ dbt.type_string() }}) as consent__consent_version
, cast(null as array) as consent__consent_scopes
, cast(null as array) as consent__domains_applied
, cast(null as {{ dbt.type_boolean() }}) as consent__gdpr_applies
{% endif %}
{% endmacro %}
24 changes: 17 additions & 7 deletions macros/field_extractions/get_cwv_fields.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,23 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0

{% macro snowflake__get_cwv_fields() %}
{% if var('snowplow__enable_cwv', false) %}
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:lcp::decimal(14,4) as cwv__lcp
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fcp::decimal(14,4) as cwv__fcp
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fid::decimal(14,4) as cwv__fid
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:cls::decimal(14,4) as cwv__cls
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:inp::decimal(14,4) as cwv__inp
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:ttfb::decimal(14,4) as cwv__ttfb
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:navigationType::varchar as cwv__navigation_type
{% if var('snowplow__snowflake_lakeloader', false) %}
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:lcp::decimal(14,4) as cwv__lcp
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fcp::decimal(14,4) as cwv__fcp
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fid::decimal(14,4) as cwv__fid
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:cls::decimal(14,4) as cwv__cls
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:inp::decimal(14,4) as cwv__inp
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:ttfb::decimal(14,4) as cwv__ttfb
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:navigation_type::varchar as cwv__navigation_type
{% else %}
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:lcp::decimal(14,4) as cwv__lcp
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fcp::decimal(14,4) as cwv__fcp
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fid::decimal(14,4) as cwv__fid
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:cls::decimal(14,4) as cwv__cls
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:inp::decimal(14,4) as cwv__inp
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:ttfb::decimal(14,4) as cwv__ttfb
, unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:navigationType::varchar as cwv__navigation_type
{% endif %}
{% else %}
, cast(null as decimal(14,4)) as cwv__lcp,
, cast(null as decimal(14,4)) as cwv__fcp,
Expand Down
24 changes: 17 additions & 7 deletions macros/field_extractions/get_geo_context_fields.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,23 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0

{% macro snowflake__get_geo_context_fields() %}
{% if var('snowplow__enable_geolocation_context', false) %}
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitude::float AS geo__latitude
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:longitude::float AS geo__longitude
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitudeLongitudeAccuracy::float AS geo__latitude_longitude_accuracy
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitude::float AS geo__altitude
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitudeAccuracy::float AS geo__altitude_accuracy
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:bearing::float AS geo__bearing
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:speed::float AS geo__speed
{% if var('snowplow__snowflake_lakeloader', false) %}
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitude::float AS geo__latitude
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:longitude::float AS geo__longitude
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitude_longitude_accuracy::float AS geo__latitude_longitude_accuracy
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitude::float AS geo__altitude
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitude_accuracy::float AS geo__altitude_accuracy
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:bearing::float AS geo__bearing
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:speed::float AS geo__speed
{% else %}
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitude::float AS geo__latitude
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:longitude::float AS geo__longitude
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:latitudeLongitudeAccuracy::float AS geo__latitude_longitude_accuracy
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitude::float AS geo__altitude
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:altitudeAccuracy::float AS geo__altitude_accuracy
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:bearing::float AS geo__bearing
, contexts_com_snowplowanalytics_snowplow_geolocation_context_1[0]:speed::float AS geo__speed
{% endif %}
{% else %}
, cast(null as {{ dbt.type_float() }}) as geo__latitude
, cast(null as {{ dbt.type_float() }}) as geo__longitude
Expand Down
Loading

0 comments on commit 6f8d381

Please sign in to comment.