✨ dbt-fabric adapter compatibility

macros/edr/dbt_artifacts/upload_dbt_invocation.sql

@@ -17,7 +17,7 @@
       'command': flags.WHICH,
       'dbt_version': dbt_version,
       'elementary_version': elementary.get_elementary_package_version(),
-      'full_refresh': flags.FULL_REFRESH,
+      'full_refresh': edr_evaluate_bool(flags.FULL_REFRESH),
       'invocation_vars': elementary.get_invocation_vars(),
       'vars': elementary.get_all_vars(),
       'target_name': target.name,

macros/edr/dbt_artifacts/upload_run_results.sql

@@ -55,7 +55,7 @@
         'execute_completed_at': none,
         'compile_started_at': none,
         'compile_completed_at': none,
-        'full_refresh': flags.FULL_REFRESH,
+        'full_refresh': edr_evaluate_bool(flags.FULL_REFRESH),
         'compiled_code': elementary.get_compiled_code(node, as_column_value=true),
         'failures': run_result_dict.get('failures'),
         'query_id': run_result_dict.get('adapter_response', {}).get('query_id'),

macros/edr/system/system_utils/empty_table.sql

@@ -131,7 +131,11 @@
 {% endmacro %}
 
 
-{% macro dummy_values() %}
+{%- macro dummy_values() -%}
+    {{ return(adapter.dispatch('dummy_values', 'elementary')()) }}
+{%- endmacro -%}
+
+{% macro default__dummy_values() %}
 
     {%- set dummy_values = {
      'string': "dummy_string",
@@ -146,3 +150,19 @@
     {{ return(dummy_values) }}
 
 {% endmacro %}
+
+{% macro fabric__dummy_values() %}
+
+    {%- set dummy_values = {
+     'string': "dummy_string",
+     'long_string': "this_is_just_a_long_dummy_string",
+     'boolean': '1',
+     'int': 123456789,
+     'bigint': 31474836478,
+     'float': 123456789.99,
+     'timestamp': "2063-04-05"
+    } %}
+
+    {{ return(dummy_values) }}
+
+{% endmacro %}

macros/utils/cross_db_utils/current_timestamp.sql

@@ -62,3 +62,11 @@
 {% macro trino__edr_current_timestamp_in_utc() -%}
     cast(current_timestamp at time zone 'UTC' as timestamp(6))
 {%- endmacro -%}
+
+{% macro fabric__edr_current_timestamp() %}
+    cast(current_timestamp as datetime2)
+{% endmacro %}
+
+{% macro fabric__edr_current_timestamp_in_utc() %}
+    cast(sysutcdatetime() as datetime2)
+{% endmacro %}

macros/utils/cross_db_utils/day_of_week.sql

@@ -40,3 +40,7 @@
 {% macro trino__edr_day_of_week_expression(date_expr) %}
     date_format({{ date_expr }}, '%W')
 {% endmacro %}
+
+{% macro fabric__edr_day_of_week_expression(date_expr) %}
+    format({{ date_expr }}, 'dddd')
+{% endmacro %}

macros/utils/cross_db_utils/hour_of_day.sql

@@ -22,3 +22,7 @@
 {% macro snowflake__edr_hour_of_day_expression(date_expr) %}
     HOUR({{ date_expr }})
 {% endmacro %}
+
+{% macro fabric__edr_hour_of_day_expression(date_expr) %}
+    DATEPART(HOUR, {{ date_expr }})
+{% endmacro %}

macros/utils/cross_db_utils/hour_of_week.sql

@@ -38,3 +38,8 @@
 {% macro trino__edr_hour_of_week_expression(date_expr) %}
     date_format({{ date_expr }}, '%W%H')
 {% endmacro %}
+
+{% macro fabric__edr_hour_of_week_expression(date_expr) %}
+    format({{ date_expr }}, 'ddddHH')
+{% endmacro %}
+

macros/utils/cross_db_utils/stddev.sql

@@ -0,0 +1,12 @@
+{# Standard deviation made to work on all adapters #}
+{% macro edr_stddev(metric_value) -%}
+    {{ return(adapter.dispatch('edr_stddev', 'elementary') (metric_value)) }}
+{%- endmacro %}
+
+{% macro default__edr_stddev(metric_value) %}
+    stddev(metric_value)
+{% endmacro %}
+
+{% macro fabric__edr_stddev(metric_value) %}
+   stdev(metric_value)
+{% endmacro %}

macros/utils/cross_db_utils/time_trunc.sql

@@ -9,4 +9,8 @@
 
 {% macro bigquery__edr_time_trunc(date_part, date_expression) %}
     timestamp_trunc(cast({{ date_expression }} as timestamp), {{ date_part }})
+{% endmacro %}
+
+{% macro fabric__edr_time_trunc(date_part, date_expression) %}
+    datetrunc({{date_part}}, cast({{ date_expression }} as {{ elementary.edr_type_timestamp() }}))
 {% endmacro %}

macros/utils/data_types/bool_type_values.sql

@@ -0,0 +1,55 @@
+{# Evaluate bool - For jinja bool true/false evaluation towards DBs compatible bool type#}
+{% macro edr_evaluate_bool(return_value) -%}
+    {%- if return_value == true -%} {{ edr_bool_true() }}   
+    {%- else -%} {{ edr_bool_false() }}
+    {%- endif -%}
+{%- endmacro %}
+
+{% macro edr_evaluate_bool_condition(return_value) -%}
+    {%- if return_value == true -%} {{ edr_bool_true_condition() }}   
+    {%- else -%} {{ edr_bool_false_condition() }}
+    {%- endif -%}
+{%- endmacro %}
+
+{# Due to "faking" of bool using bit in T-SQL we have distinguish representation of true/false #}
+{# We have different bools for use in parameter wheres vs. in columns as values#}
+
+
+{#True bools#}
+
+{% macro edr_bool_true() -%}
+    {{ return(adapter.dispatch('edr_bool_true', 'elementary')()) }}
+{%- endmacro %}
+
+{% macro default__edr_bool_true() %}true{% endmacro %}
+
+{% macro fabric__edr_bool_true() %}1{% endmacro %}
+
+
+{% macro edr_bool_true_condition() -%}
+    {{ return(adapter.dispatch('edr_bool_true_condition', 'elementary')()) }}
+{%- endmacro %}
+
+{% macro default__edr_bool_true_condition() %}true{% endmacro %}
+
+{% macro fabric__edr_bool_true_condition() %}1 = 1{% endmacro %}
+
+{# False bools#} 
+
+{% macro edr_bool_false() -%}
+    {{ return(adapter.dispatch('edr_bool_false', 'elementary')()) }}
+{%- endmacro %}
+
+{% macro default__edr_bool_false() %}false{% endmacro %}
+
+{% macro fabric__edr_bool_false() %}0{% endmacro %}
+
+
+
+{% macro edr_bool_false_condition() -%}
+    {{ return(adapter.dispatch('edr_bool_false_condition', 'elementary')()) }}
+{%- endmacro %}
+
+{% macro default__edr_bool_false_condition() %}false{% endmacro %}
+
+{% macro fabric__edr_bool_false_condition() %}0 = 1{% endmacro %}

macros/utils/data_types/data_type.sql

@@ -19,6 +19,10 @@
     {% do return("BOOL") %}
 {% endmacro %}
 
+{% macro fabric__edr_type_bool() %}
+    {% do return("bit") %}
+{% endmacro %}
+
 
 {%- macro edr_type_string() -%}
     {{ return(adapter.dispatch('edr_type_string', 'elementary')()) }}
@@ -59,6 +63,9 @@
     {% do return("varchar") %}
 {% endmacro %}
 
+{% macro fabric__edr_type_string() %}
+    {% do return("varchar(4096)") %}
+{% endmacro %}
 
 
 
@@ -152,3 +159,7 @@
 {% macro trino__edr_type_timestamp() %}
     timestamp(6)
 {% endmacro %}
+
+{% macro fabric__edr_type_timestamp() %}
+    datetime2(2)
+{% endmacro %}

macros/utils/data_types/data_type_list.sql

@@ -131,3 +131,24 @@
     {%- endif %}
 
 {% endmacro %}
+
+{% macro fabric__data_type_list(data_type) %}
+
+    {% set string_list = ['varchar', 'char'] | list %}
+    {% set numeric_list = ['smallint', 'int', 'bigint', 'float', 'real', 'numeric'] | list %}
+    {% set timestamp_list = ['datetime2','date', 'time'] | list %}
+    {% set boolean_list = ["bit"] | list %}
+
+    {%- if data_type == 'string' %}
+        {{ return(string_list) }}
+    {%- elif data_type == 'numeric' %}
+        {{ return(numeric_list) }}
+    {%- elif data_type == 'timestamp' %}
+        {{ return(timestamp_list) }}
+    {%- elif data_type == "boolean" %}
+        {{ return(boolean_list) }}
+    {%- else %}
+        {{ return([]) }}
+    {%- endif %}
+
+{% endmacro %}

macros/utils/table_operations/has_temp_table_support.sql

@@ -18,3 +18,6 @@
     {% do return(false) %}
 {% endmacro %}
 
+{% macro fabric__has_temp_table_support() %}
+    {% do return(false) %}
+{% endmacro %}

macros/utils/table_operations/insert_rows.sql

@@ -157,6 +157,10 @@
     {{- return(string_value | replace("'", "''")) -}}
 {%- endmacro -%}
 
+{%- macro fabric__escape_special_chars(string_value) -%}
+    {{- return(string_value | replace("'", "''")) -}}
+{%- endmacro -%}
+
 {%- macro render_value(value, data_type) -%}
     {%- if value is defined and value is not none -%}
         {%- if value is number -%}

models/edr/alerts/alerts_anomaly_detection.sql

@@ -34,7 +34,7 @@ alerts_anomaly_detection as (
            status,
            result_rows
         from elementary_test_results
-        where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'anomaly_detection'
+        where {{ edr_evaluate_bool_condition ( not elementary.get_config_var('disable_test_alerts')) }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'anomaly_detection'
 )
 
 select * from alerts_anomaly_detection

models/edr/alerts/alerts_dbt_models.sql

@@ -76,4 +76,4 @@ select model_execution_id as alert_id,
        status,
        full_refresh
 from error_models
-where {{ not elementary.get_config_var('disable_model_alerts') }} and lower(status) != 'success' {%- if elementary.get_config_var('disable_skipped_model_alerts') -%} and lower(status) != 'skipped' {%- endif -%}
+where {{ edr_evaluate_bool_condition ( not elementary.get_config_var('disable_model_alerts')) }} and lower(status) != 'success' {%- if elementary.get_config_var('disable_skipped_model_alerts') -%} and lower(status) != 'skipped' {%- endif -%}

models/edr/alerts/alerts_dbt_source_freshness.sql

@@ -43,4 +43,4 @@ select
   results.filter as freshness_filter
 from results
 join sources on results.unique_id = sources.unique_id
-where {{ not elementary.get_config_var('disable_source_freshness_alerts') }} and lower(status) != 'pass'
+where {{ edr_evaluate_bool_condition ( not elementary.get_config_var('disable_source_freshness_alerts')) }} and lower(status) != 'pass'

models/edr/alerts/alerts_dbt_tests.sql

@@ -34,7 +34,11 @@ alerts_dbt_tests as (
            status,
            result_rows
         from elementary_test_results
-        where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {% if elementary.get_config_var('disable_warn_alerts') %} and lower(status) != 'warn' {% endif %} {% if elementary.get_config_var('disable_skipped_test_alerts') %} and lower(status) != 'skipped' {% endif %} and test_type = 'dbt_test'
+        where {{ edr_evaluate_bool_condition (not elementary.get_config_var('disable_test_alerts')) }} 
+        and lower(status) != 'pass' 
+        {% if elementary.get_config_var('disable_warn_alerts') %} and lower(status) != 'warn' {% endif %}
+        {% if elementary.get_config_var('disable_skipped_test_alerts') %} and lower(status) != 'skipped' {% endif %} 
+        and test_type = 'dbt_test'
 )
 
 select * from alerts_dbt_tests

models/edr/alerts/alerts_schema_changes.sql

@@ -35,7 +35,7 @@ alerts_schema_changes as (
            status,
            result_rows
         from elementary_test_results
-        where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'schema_change'
+        where {{ edr_evaluate_bool_condition ( not elementary.get_config_var('disable_test_alerts')) }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'schema_change'
 )
 
 select * from alerts_schema_changes

models/edr/data_monitoring/anomaly_detection/anomaly_threshold_sensitivity.sql

@@ -21,13 +21,13 @@ score_sensitivity as (
         training_avg as metric_avg,
         training_stddev as metric_stddev,
         anomaly_score,
-        case when abs(anomaly_score) >= 1.5 then true else false end as {{ elementary.edr_quote_column('is_anomaly_1_5') }},
-        case when abs(anomaly_score) >= 2 then true else false end as {{ elementary.edr_quote_column('is_anomaly_2') }},
-        case when abs(anomaly_score) >= 2.5 then true else false end as {{ elementary.edr_quote_column('is_anomaly_2_5') }},
-        case when abs(anomaly_score) >= 3 then true else false end as {{ elementary.edr_quote_column('is_anomaly_3') }},
-        case when abs(anomaly_score) >= 3.5 then true else false end as {{ elementary.edr_quote_column('is_anomaly_3_5') }},
-        case when abs(anomaly_score) >= 4 then true else false end as {{ elementary.edr_quote_column('is_anomaly_4') }},
-        case when abs(anomaly_score) >= 4.5 then true else false end as {{ elementary.edr_quote_column('is_anomaly_4_5') }}
+        case when abs(anomaly_score) >= 1.5 then {{ elementary.edr_bool_true() }} else {{ elementary.edr_bool_false() }} end as {{ elementary.edr_quote_column('is_anomaly_1_5') }},
+        case when abs(anomaly_score) >= 2 then {{ elementary.edr_bool_true() }} else {{ elementary.edr_bool_false() }} end as {{ elementary.edr_quote_column('is_anomaly_2') }},
+        case when abs(anomaly_score) >= 2.5 then {{ elementary.edr_bool_true() }} else {{ elementary.edr_bool_false() }} end as {{ elementary.edr_quote_column('is_anomaly_2_5') }},
+        case when abs(anomaly_score) >= 3 then {{ elementary.edr_bool_true() }} else {{ elementary.edr_bool_false() }} end as {{ elementary.edr_quote_column('is_anomaly_3') }},
+        case when abs(anomaly_score) >= 3.5 then {{ elementary.edr_bool_true() }} else {{ elementary.edr_bool_false() }} end as {{ elementary.edr_quote_column('is_anomaly_3_5') }},
+        case when abs(anomaly_score) >= 4 then {{ elementary.edr_bool_true() }} else {{ elementary.edr_bool_false() }} end as {{ elementary.edr_quote_column('is_anomaly_4') }},
+        case when abs(anomaly_score) >= 4.5 then {{ elementary.edr_bool_true() }} else {{ elementary.edr_bool_false() }} end as {{ elementary.edr_quote_column('is_anomaly_4_5') }}
     from metrics_anomaly_score
     where abs(anomaly_score) >= 1.5
 

models/edr/data_monitoring/anomaly_detection/metrics_anomaly_score.sql

@@ -27,12 +27,24 @@ time_window_aggregation as (
         bucket_duration_hours,
         updated_at,
         avg(metric_value) over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) as training_avg,
-        stddev(metric_value) over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) as training_stddev,
+        {{ elementary.edr_stddev('metric_value') }} over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) as training_stddev,
         count(metric_value) over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) as training_set_size,
         last_value(bucket_end) over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) training_end,
         first_value(bucket_end) over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) as training_start
     from data_monitoring_metrics
-    {{ dbt_utils.group_by(12) }}
+    group by
+        id,
+        full_table_name,
+        column_name,
+        dimension,
+        dimension_value,
+        metric_name,
+        metric_value,
+        source_value,
+        bucket_start,
+        bucket_end,
+        bucket_duration_hours,
+        updated_at
 ),
 
 metrics_anomaly_score as (
@@ -63,9 +75,21 @@ metrics_anomaly_score as (
             metric_value is not null
             and training_avg is not null
             and bucket_end >= {{ elementary.edr_timeadd('day', '-7', elementary.edr_date_trunc('day', elementary.edr_current_timestamp())) }}
-    {{ dbt_utils.group_by(15) }}
-    order by bucket_end desc
-
+    group by
+        id,
+        full_table_name,
+        column_name,
+        dimension,
+        dimension_value,
+        metric_name,
+        metric_value,
+        bucket_start,
+        bucket_end,
+        training_avg,
+        training_stddev,
+        training_start,
+        training_end,
+        training_set_size
 
 ),
 
@@ -89,8 +113,10 @@ final as (
         training_set_size,
         updated_at,
         case
-            when abs(anomaly_score) > {{ elementary.get_config_var('anomaly_sensitivity') }} then true
-            else false end
+            when abs(anomaly_score) > {{ elementary.get_config_var('anomaly_sensitivity') }} 
+                then {{ elementary.edr_bool_true() }}
+                else {{ elementary.edr_bool_false() }}
+            end
         as is_anomaly
     from metrics_anomaly_score
 )

models/edr/dbt_artifacts/dbt_artifacts_hashes.sql

@@ -23,4 +23,3 @@ select
 from {{ ref(artifact_model) }}
 {% if not loop.last %} union all {% endif %}
 {% endfor %}
-order by metadata_hash

models/edr/run_results/model_run_results.sql

@@ -42,12 +42,12 @@ SELECT
     models.alias,
     ROW_NUMBER() OVER (PARTITION BY run_results.unique_id ORDER BY run_results.generated_at DESC) AS model_invocation_reverse_index,
     CASE WHEN FIRST_VALUE(invocation_id) OVER (PARTITION BY {{ elementary.edr_time_trunc('day', 'run_results.generated_at') }} ORDER BY run_results.generated_at ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) = invocation_id
-              THEN TRUE
-              ELSE FALSE 
+              THEN {{ elementary.edr_bool_true() }}
+              ELSE {{ elementary.edr_bool_false() }}
          END                                                               AS is_the_first_invocation_of_the_day,
     CASE WHEN LAST_VALUE(invocation_id) OVER (PARTITION BY {{ elementary.edr_time_trunc('day', 'run_results.generated_at') }} ORDER BY run_results.generated_at ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) = invocation_id
-              THEN TRUE
-              ELSE FALSE 
+              THEN {{ elementary.edr_bool_true() }}
+              ELSE {{ elementary.edr_bool_false() }}
          END                                                               AS is_the_last_invocation_of_the_day
 
 FROM dbt_run_results run_results