Release/snowplow unified/0.4.4

.github/workflows/pr_tests.yml

@@ -60,6 +60,7 @@ jobs:
         # Run tests from integration_tests sub dir
         working-directory: ./integration_tests
     strategy:
+      fail-fast: false
       matrix:
         dbt_version: ["1.*"]
         warehouse: ["postgres", "bigquery", "snowflake", "databricks", "redshift"] # TODO: Add RS self-hosted runner

CHANGELOG

@@ -1,3 +1,20 @@
+snowplow-unified 0.4.4 (2024-06-26)
+---------------------------------------
+## Summary
+This release adds early-release support for running the models on a Snowflake lake loader Iceberg table. Please note this feature is actively in testing and not all features of the package may work as expected. Please [let us know](https://github.com/snowplow/dbt-snowplow-unified/issues/new/choose) if you find any issues.
+
+## Features
+- Add early release support for Snowflake Iceberg events tables via lake loader
+
+# Optimizations
+- Remove unused columns and attempt to improve window functions in user_sessions_this_run
+
+## Under the hood
+- Alter GH action to remove fail-fast on integration tests
+
+## Upgrading
+Update the snowplow-unified version in your `packages.yml` file.
+
 snowplow-unified 0.4.3 (2024-06-24)
 ---------------------------------------
 ## Summary

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'snowplow_unified'
-version: '0.4.3'
+version: '0.4.4'
 config-version: 2
 
 require-dbt-version: [">=1.6.0", "<2.0.0"]
@@ -115,6 +115,7 @@ vars:
 
     # Snowflake:
     snowplow__query_tag: snowplow_dbt
+    snowplow__snowflake_lakeloader: false
 
     # Databricks:
     # Depending on the use case it should either be the catalog (for Unity Catalog users from databricks connector 1.1.1 onwards) or the same value as your snowplow__atomic_schema (unless changed it should be 'atomic')

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'snowplow_unified_integration_tests'
-version: '0.4.3'
+version: '0.4.4'
 config-version: 2
 
 profile: 'integration_tests'

macros/field_extractions/get_app_error_event_fields.sql

@@ -84,15 +84,28 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 
 {% macro snowflake__get_app_error_event_fields() %}
   {% if var('snowplow__enable_app_errors', false) %}
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:message::VARCHAR() AS app_error__message
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:programmingLanguage::VARCHAR() AS app_error__programming_language
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:className::VARCHAR() AS app_error__class_name
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:exceptionName::VARCHAR() AS app_error__exception_name
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:isFatal::BOOLEAN AS app_error__is_fatal
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:lineNumber::INT AS app_error__line_number
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:stackTrace::VARCHAR() AS app_error__stack_trace
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadId::INT AS app_error__thread_id
-    , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadName::VARCHAR() AS app_error__thread_name
+    {% if var('snowplow__snowflake_lakeloader', false) %}
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:message::VARCHAR() AS app_error__message
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:programming_language::VARCHAR() AS app_error__programming_language
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:class_name::VARCHAR() AS app_error__class_name
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:exception_name::VARCHAR() AS app_error__exception_name
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:is_fatal::BOOLEAN AS app_error__is_fatal
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:line_number::INT AS app_error__line_number
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:stack_trace::VARCHAR() AS app_error__stack_trace
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:thread_id::INT AS app_error__thread_id
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:thread_name::VARCHAR() AS app_error__thread_name
+    {% else %}
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:message::VARCHAR() AS app_error__message
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:programmingLanguage::VARCHAR() AS app_error__programming_language
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:className::VARCHAR() AS app_error__class_name
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:exceptionName::VARCHAR() AS app_error__exception_name
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:isFatal::BOOLEAN AS app_error__is_fatal
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:lineNumber::INT AS app_error__line_number
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:stackTrace::VARCHAR() AS app_error__stack_trace
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadId::INT AS app_error__thread_id
+      , unstruct_event_com_snowplowanalytics_snowplow_application_error_1:threadName::VARCHAR() AS app_error__thread_name
+    {% endif %}
+
   {% else %}
     , cast(null as {{ dbt.type_string() }}) as app_error__message
     , cast(null as {{ dbt.type_string() }}) as app_error__programming_language

macros/field_extractions/get_browser_context_fields.sql

@@ -105,19 +105,35 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 
 {% macro snowflake__get_browser_context_fields() %}
   {% if var('snowplow__enable_browser_context', false) %}
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:viewport::varchar AS browser__viewport
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentSize::varchar AS browser__document_size
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:resolution::varchar AS browser__resolution
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:colorDepth::int AS browser__color_depth
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:devicePixelRatio::float AS browser__device_pixel_ratio
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:cookiesEnabled::boolean AS browser__cookies_enabled
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:online::boolean AS browser__online
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:browserLanguage::varchar AS browser__browser_language
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentLanguage::varchar AS browser__document_language
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:webdriver::boolean AS browser__webdriver
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:deviceMemory::int AS browser__device_memory
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:hardwareConcurrency::int AS browser__hardware_concurrency
-    , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:tabId::varchar AS browser__tab_id
+    {% if var('snowplow__snowflake_lakeloader', false) %}
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:viewport::varchar AS browser__viewport
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:document_size::varchar AS browser__document_size
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:resolution::varchar AS browser__resolution
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:color_depth::int AS browser__color_depth
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:device_pixel_ratio::float AS browser__device_pixel_ratio
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:cookies_enabled::boolean AS browser__cookies_enabled
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:online::boolean AS browser__online
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:browser_language::varchar AS browser__browser_language
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:document_language::varchar AS browser__document_language
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:webdriver::boolean AS browser__webdriver
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:device_memory::int AS browser__device_memory
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:hardware_concurrency::int AS browser__hardware_concurrency
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:tab_id::varchar AS browser__tab_id
+    {% else %}
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:viewport::varchar AS browser__viewport
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentSize::varchar AS browser__document_size
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:resolution::varchar AS browser__resolution
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:colorDepth::int AS browser__color_depth
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:devicePixelRatio::float AS browser__device_pixel_ratio
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:cookiesEnabled::boolean AS browser__cookies_enabled
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:online::boolean AS browser__online
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:browserLanguage::varchar AS browser__browser_language
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:documentLanguage::varchar AS browser__document_language
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:webdriver::boolean AS browser__webdriver
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:deviceMemory::int AS browser__device_memory
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:hardwareConcurrency::int AS browser__hardware_concurrency
+      , contexts_com_snowplowanalytics_snowplow_browser_context_1[0]:tabId::varchar AS browser__tab_id
+    {% endif %}
   {% else %}
     , cast(null as {{ dbt.type_string() }}) as browser__viewport
     , cast(null as {{ dbt.type_string() }}) as browser__document_size

macros/field_extractions/get_cmp_visible_event_fields.sql

@@ -46,7 +46,11 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 
 {% macro snowflake__get_cmp_visible_event_fields() %}
     {% if var('snowplow__enable_consent', false) %}
-    , unstruct_event_com_snowplowanalytics_snowplow_cmp_visible_1:elapsedTime::float as cmp__elapsed_time
+      {% if var('snowplow__snowflake_lakeloader', false) %}
+        , unstruct_event_com_snowplowanalytics_snowplow_cmp_visible_1:elapsed_time::float as cmp__elapsed_time
+      {% else %}
+        , unstruct_event_com_snowplowanalytics_snowplow_cmp_visible_1:elapsedTime::float as cmp__elapsed_time
+      {% endif %}
     {% else %}
       , cast(null as {{ dbt.type_float() }}) as cmp__elapsed_time
     {% endif %}

macros/field_extractions/get_consent_event_fields.sql

@@ -74,23 +74,32 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 {% endmacro %}
 
 {% macro snowflake__get_consent_event_fields() %}
-    {% if var('snowplow__enable_consent', false) %}
-
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:eventType::varchar as consent__event_type
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:basisForProcessing::varchar as consent__basis_for_processing
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentUrl::varchar as consent__consent_url
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentVersion::varchar as consent__consent_version
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentScopes::array as consent__consent_scopes
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:domainsApplied::array as consent__domains_applied
-    , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:gdprApplies::boolean as consent__gdpr_applies
-
+  {% if var('snowplow__enable_consent', false) %}
+    {% if var('snowplow__snowflake_lakeloader', false) %}
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:event_type::varchar as consent__event_type
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:basis_for_processing::varchar as consent__basis_for_processing
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consent_url::varchar as consent__consent_url
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consent_version::varchar as consent__consent_version
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consent_scopes::array as consent__consent_scopes
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:domains_applied::array as consent__domains_applied
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:gdpr_applies::boolean as consent__gdpr_applies
     {% else %}
-      , cast(null as {{ dbt.type_string() }}) as consent__event_type
-      , cast(null as {{ dbt.type_string() }}) as consent__basis_for_processing
-      , cast(null as {{ dbt.type_string() }}) as consent__consent_url
-      , cast(null as {{ dbt.type_string() }}) as consent__consent_version
-      , cast(null as array) as consent__consent_scopes
-      , cast(null as array) as consent__domains_applied
-      , cast(null as {{ dbt.type_boolean() }}) as consent__gdpr_applies
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:eventType::varchar as consent__event_type
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:basisForProcessing::varchar as consent__basis_for_processing
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentUrl::varchar as consent__consent_url
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentVersion::varchar as consent__consent_version
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:consentScopes::array as consent__consent_scopes
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:domainsApplied::array as consent__domains_applied
+      , unstruct_event_com_snowplowanalytics_snowplow_consent_preferences_1:gdprApplies::boolean as consent__gdpr_applies
     {% endif %}
+
+  {% else %}
+    , cast(null as {{ dbt.type_string() }}) as consent__event_type
+    , cast(null as {{ dbt.type_string() }}) as consent__basis_for_processing
+    , cast(null as {{ dbt.type_string() }}) as consent__consent_url
+    , cast(null as {{ dbt.type_string() }}) as consent__consent_version
+    , cast(null as array) as consent__consent_scopes
+    , cast(null as array) as consent__domains_applied
+    , cast(null as {{ dbt.type_boolean() }}) as consent__gdpr_applies
+  {% endif %}
 {% endmacro %}

macros/field_extractions/get_cwv_fields.sql

@@ -74,13 +74,23 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 
 {% macro snowflake__get_cwv_fields() %}
     {% if var('snowplow__enable_cwv', false) %}
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:lcp::decimal(14,4) as cwv__lcp
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fcp::decimal(14,4) as cwv__fcp
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fid::decimal(14,4) as cwv__fid
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:cls::decimal(14,4) as cwv__cls
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:inp::decimal(14,4) as cwv__inp
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:ttfb::decimal(14,4) as cwv__ttfb
-    , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:navigationType::varchar as cwv__navigation_type
+      {% if var('snowplow__snowflake_lakeloader', false) %}
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:lcp::decimal(14,4) as cwv__lcp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fcp::decimal(14,4) as cwv__fcp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fid::decimal(14,4) as cwv__fid
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:cls::decimal(14,4) as cwv__cls
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:inp::decimal(14,4) as cwv__inp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:ttfb::decimal(14,4) as cwv__ttfb
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:navigation_type::varchar as cwv__navigation_type
+      {% else %}
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:lcp::decimal(14,4) as cwv__lcp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fcp::decimal(14,4) as cwv__fcp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:fid::decimal(14,4) as cwv__fid
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:cls::decimal(14,4) as cwv__cls
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:inp::decimal(14,4) as cwv__inp
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:ttfb::decimal(14,4) as cwv__ttfb
+      , unstruct_event_com_snowplowanalytics_snowplow_web_vitals_1:navigationType::varchar as cwv__navigation_type
+      {% endif %}
     {% else %}
     , cast(null as decimal(14,4)) as cwv__lcp,
     , cast(null as decimal(14,4)) as cwv__fcp,