diff --git a/integration_tests/tests/dbt_project.py b/integration_tests/tests/dbt_project.py index bb2f026b3..e9adefa02 100644 --- a/integration_tests/tests/dbt_project.py +++ b/integration_tests/tests/dbt_project.py @@ -42,7 +42,7 @@ def get_dbt_runner(target: str, project_dir: str) -> BaseDbtRunner: class DbtProject: def __init__(self, target: str, project_dir: str): self.dbt_runner = get_dbt_runner(target, project_dir) - + self.target = target self.project_dir_path = Path(project_dir) self.models_dir_path = self.project_dir_path / "models" self.tmp_models_dir_path = self.models_dir_path / "tmp" @@ -57,8 +57,8 @@ def run_query(self, prerendered_query: str): ) return results - @staticmethod def read_table_query( + self, table_name: str, where: Optional[str] = None, group_by: Optional[str] = None, @@ -66,14 +66,23 @@ def read_table_query( limit: Optional[int] = None, column_names: Optional[List[str]] = None, ): - return f""" - SELECT {', '.join(column_names) if column_names else '*'} - FROM {{{{ ref('{table_name}') }}}} - {f"WHERE {where}" if where else ""} - {f"GROUP BY {group_by}" if group_by else ""} - {f"ORDER BY {order_by}" if order_by else ""} - {f"LIMIT {limit}" if limit else ""} - """ + if self.target == "fabric": + return f""" + SELECT {f'TOP {limit}' if limit else ''} {', '.join(column_names) if column_names else '*'} + FROM {{{{ ref('{table_name}') }}}} + {f"WHERE {where}" if where else ""} + {f"GROUP BY {group_by}" if group_by else ""} + {f"ORDER BY {order_by}" if order_by else ""} + """ + else: + return f""" + SELECT {', '.join(column_names) if column_names else '*'} + FROM {{{{ ref('{table_name}') }}}} + {f"WHERE {where}" if where else ""} + {f"GROUP BY {group_by}" if group_by else ""} + {f"ORDER BY {order_by}" if order_by else ""} + {f"LIMIT {limit}" if limit else ""} + """ def read_table( self, diff --git a/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql b/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql index d822c5110..19cff62df 100644 --- a/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql +++ b/macros/edr/data_monitoring/anomaly_detection/get_anomaly_scores_query.sql @@ -1,4 +1,8 @@ {% macro get_anomaly_scores_query(test_metrics_table_relation, model_relation, test_configuration, metric_names, column_name = none, columns_only = false, metric_properties = none, data_monitoring_metrics_table=none) %} + {{ return(adapter.dispatch('get_anomaly_scores_query', 'elementary')(test_metrics_table_relation, model_relation, test_configuration, metric_names, column_name, columns_only, metric_properties, data_monitoring_metrics_table)) }} +{% endmacro %} + +{% macro default__get_anomaly_scores_query(test_metrics_table_relation, model_relation, test_configuration, metric_names, column_name = none, columns_only = false, metric_properties = none, data_monitoring_metrics_table=none) %} {%- set model_graph_node = elementary.get_model_graph_node(model_relation) %} {%- set full_table_name = elementary.model_node_to_full_name(model_graph_node) %} {%- set test_execution_id = elementary.get_test_execution_id() %} @@ -227,37 +231,3 @@ {% endset %} {{ return(anomaly_scores_query) }} {% endmacro %} - -{% macro get_negative_value_supported_metrics() %} - {% do return(["min", "max", "average", "standard_deviation", "variance", "sum"]) %} -{% endmacro %} - -{% macro get_limit_metric_values(test_configuration) %} - {%- set min_val -%} - ((-1) * {{ test_configuration.anomaly_sensitivity }} * training_stddev + training_avg) - {%- endset -%} - - {% if test_configuration.ignore_small_changes.drop_failure_percent_threshold %} - {%- set drop_avg_threshold -%} - ((1 - {{ test_configuration.ignore_small_changes.drop_failure_percent_threshold }}/100.0) * training_avg) - {%- endset -%} - {%- set min_val -%} - {{ elementary.arithmetic_min(drop_avg_threshold, min_val) }} - {%- endset -%} - {% endif %} - - {%- set max_val -%} - ({{ test_configuration.anomaly_sensitivity }} * training_stddev + training_avg) - {%- endset -%} - - {% if test_configuration.ignore_small_changes.spike_failure_percent_threshold %} - {%- set spike_avg_threshold -%} - ((1 + {{ test_configuration.ignore_small_changes.spike_failure_percent_threshold }}/100.0) * training_avg) - {%- endset -%} - {%- set max_val -%} - {{ elementary.arithmetic_max(spike_avg_threshold, max_val) }} - {%- endset -%} - {% endif %} - - {{ return({"min_metric_value": min_val, "max_metric_value": max_val}) }} -{% endmacro %} diff --git a/macros/edr/data_monitoring/data_monitors_configuration/get_buckets_configuration.sql b/macros/edr/data_monitoring/data_monitors_configuration/get_buckets_configuration.sql index 1d66953bb..dfe8e44ae 100644 --- a/macros/edr/data_monitoring/data_monitors_configuration/get_buckets_configuration.sql +++ b/macros/edr/data_monitoring/data_monitors_configuration/get_buckets_configuration.sql @@ -14,7 +14,7 @@ {{ return(trunc_min_bucket_start_expr) }} {% endmacro %} -{# This macro can't be used without truncating to full buckets #} +{# This macro cant be used without truncating to full buckets #} {% macro get_backfill_bucket_start(detection_end, backfill_days) %} {% do return((detection_end - modules.datetime.timedelta(backfill_days)).strftime("%Y-%m-%d 00:00:00")) %} {% endmacro %} @@ -40,7 +40,7 @@ {%- endif %} {%- set regular_bucket_times_query %} - with bucket_times as ( + ;with bucket_times as ( select {{ trunc_min_bucket_start_expr }} as days_back_start , {{ detection_end_expr }} as detection_end @@ -58,7 +58,7 @@ {%- endset %} {%- set incremental_bucket_times_query %} - with all_buckets as ( + ;with all_buckets as ( select edr_bucket_start as bucket_start, edr_bucket_end as bucket_end from ({{ elementary.complete_buckets_cte(metric_properties, trunc_min_bucket_start_expr, detection_end_expr) }}) results where edr_bucket_start >= {{ trunc_min_bucket_start_expr }} diff --git a/macros/edr/data_monitoring/monitors_query/column_monitoring_query.sql b/macros/edr/data_monitoring/monitors_query/column_monitoring_query.sql index 0f35f1f1b..fadc59814 100644 --- a/macros/edr/data_monitoring/monitors_query/column_monitoring_query.sql +++ b/macros/edr/data_monitoring/monitors_query/column_monitoring_query.sql @@ -173,18 +173,3 @@ from metrics_final {% endmacro %} - -{% macro select_dimensions_columns(dimension_columns, as_prefix="") %} - {% set select_statements %} - {%- for column in dimension_columns -%} - {{ column }} - {%- if as_prefix -%} - {{ " as " ~ as_prefix ~ "_" ~ column }} - {%- endif -%} - {%- if not loop.last -%} - {{ ", " }} - {%- endif -%} - {%- endfor -%} - {% endset %} - {{ return(select_statements) }} -{% endmacro %} diff --git a/macros/edr/dbt_artifacts/upload_dbt_invocation.sql b/macros/edr/dbt_artifacts/upload_dbt_invocation.sql index 93ff28473..d8955626a 100644 --- a/macros/edr/dbt_artifacts/upload_dbt_invocation.sql +++ b/macros/edr/dbt_artifacts/upload_dbt_invocation.sql @@ -17,7 +17,7 @@ 'command': flags.WHICH, 'dbt_version': dbt_version, 'elementary_version': elementary.get_elementary_package_version(), - 'full_refresh': flags.FULL_REFRESH, + 'full_refresh': edr_evaluate_bool(flags.FULL_REFRESH), 'invocation_vars': elementary.get_invocation_vars(), 'vars': elementary.get_all_vars(), 'target_name': target.name, diff --git a/macros/edr/dbt_artifacts/upload_run_results.sql b/macros/edr/dbt_artifacts/upload_run_results.sql index 9de8b253a..99127208c 100644 --- a/macros/edr/dbt_artifacts/upload_run_results.sql +++ b/macros/edr/dbt_artifacts/upload_run_results.sql @@ -36,6 +36,9 @@ {{ return(dbt_run_results_empty_table_query) }} {% endmacro %} +{% macro debug_var(var_name, var_value) %} + {% do log("🔍 DEBUG — " ~ var_name ~ ": " ~ var_value, info=True) %} +{% endmacro %} {% macro flatten_run_result(run_result) %} {% set run_result_dict = elementary.get_run_result_dict(run_result) %} {% set node = elementary.safe_get_with_default(run_result_dict, 'node', {}) %} @@ -55,7 +58,7 @@ 'execute_completed_at': none, 'compile_started_at': none, 'compile_completed_at': none, - 'full_refresh': flags.FULL_REFRESH, + 'full_refresh': edr_evaluate_bool(flags.FULL_REFRESH), 'compiled_code': elementary.get_compiled_code(node, as_column_value=true), 'failures': run_result_dict.get('failures'), 'query_id': run_result_dict.get('adapter_response', {}).get('query_id'), diff --git a/macros/edr/system/system_utils/empty_table.sql b/macros/edr/system/system_utils/empty_table.sql index 6adcd013d..3645d40a2 100644 --- a/macros/edr/system/system_utils/empty_table.sql +++ b/macros/edr/system/system_utils/empty_table.sql @@ -131,7 +131,11 @@ {% endmacro %} -{% macro dummy_values() %} +{%- macro dummy_values() -%} + {{ return(adapter.dispatch('dummy_values', 'elementary')()) }} +{%- endmacro -%} + +{% macro default__dummy_values() %} {%- set dummy_values = { 'string': "dummy_string", @@ -146,3 +150,19 @@ {{ return(dummy_values) }} {% endmacro %} + +{% macro fabric__dummy_values() %} + + {%- set dummy_values = { + 'string': "dummy_string", + 'long_string': "this_is_just_a_long_dummy_string", + 'boolean': '1', + 'int': 123456789, + 'bigint': 31474836478, + 'float': 123456789.99, + 'timestamp': "2063-04-05" + } %} + + {{ return(dummy_values) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/edr/tests/test_all_columns_anomalies.sql b/macros/edr/tests/test_all_columns_anomalies.sql index dced33e3d..4c62dfa05 100644 --- a/macros/edr/tests/test_all_columns_anomalies.sql +++ b/macros/edr/tests/test_all_columns_anomalies.sql @@ -89,11 +89,9 @@ metric_names=all_columns_monitors, columns_only=true, metric_properties=metric_properties) %} - {% set anomaly_scores_test_table_relation = elementary.create_elementary_test_table(database_name, tests_schema_name, test_table_name, 'anomaly_scores', anomaly_scores_query) %} {{- elementary.test_log('end', full_table_name, 'all columns') }} - {% set flattened_test = elementary.flatten_test(context["model"]) %} {% set anomaly_scores_sql = elementary.get_read_anomaly_scores_query() %} {% do elementary.store_metrics_table_in_cache() %} @@ -125,5 +123,4 @@ {%- endif -%} {%- endif -%} {{ return(False) }} -{%- endmacro -%} - +{%- endmacro -%} \ No newline at end of file diff --git a/macros/edr/tests/test_utils/create_elementary_test_table.sql b/macros/edr/tests/test_utils/create_elementary_test_table.sql index 28c6d3280..1670a5c30 100644 --- a/macros/edr/tests/test_utils/create_elementary_test_table.sql +++ b/macros/edr/tests/test_utils/create_elementary_test_table.sql @@ -1,5 +1,5 @@ {% macro create_elementary_test_table(database_name, schema_name, test_name, table_type, sql_query) %} - {% if execute %} + {% if execute %} {% set temp_table_name = elementary.table_name_with_suffix(test_name, "__" ~ table_type ~ elementary.get_timestamped_table_suffix()).replace("*", "") %} {% set default_identifier_quoting = api.Relation.get_default_quote_policy().get_part("identifier") %} @@ -14,7 +14,7 @@ identifier=temp_table_name, type='table') -%} - {# Create the table if it doesn't exist #} + {# Create the table if it doesnt exist #} {%- do elementary.create_or_replace(false, temp_table_relation, sql_query) %} {# Cache the test table for easy access later #} diff --git a/macros/utils/cross_db_utils/current_timestamp.sql b/macros/utils/cross_db_utils/current_timestamp.sql index 3ea59a809..6b6fc528a 100644 --- a/macros/utils/cross_db_utils/current_timestamp.sql +++ b/macros/utils/cross_db_utils/current_timestamp.sql @@ -62,3 +62,11 @@ {% macro trino__edr_current_timestamp_in_utc() -%} cast(current_timestamp at time zone 'UTC' as timestamp(6)) {%- endmacro -%} + +{% macro fabric__edr_current_timestamp() %} + cast(current_timestamp as datetime2) +{% endmacro %} + +{% macro fabric__edr_current_timestamp_in_utc() %} + cast(sysutcdatetime() as datetime2) +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/cross_db_utils/day_of_week.sql b/macros/utils/cross_db_utils/day_of_week.sql index 5c0e3d88f..1ae738969 100644 --- a/macros/utils/cross_db_utils/day_of_week.sql +++ b/macros/utils/cross_db_utils/day_of_week.sql @@ -40,3 +40,7 @@ {% macro trino__edr_day_of_week_expression(date_expr) %} date_format({{ date_expr }}, '%W') {% endmacro %} + +{% macro fabric__edr_day_of_week_expression(date_expr) %} + format({{ date_expr }}, 'dddd') +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/cross_db_utils/hour_of_day.sql b/macros/utils/cross_db_utils/hour_of_day.sql index c62ed7a7a..9f6b34631 100644 --- a/macros/utils/cross_db_utils/hour_of_day.sql +++ b/macros/utils/cross_db_utils/hour_of_day.sql @@ -22,3 +22,7 @@ {% macro snowflake__edr_hour_of_day_expression(date_expr) %} HOUR({{ date_expr }}) {% endmacro %} + +{% macro fabric__edr_hour_of_day_expression(date_expr) %} + DATEPART(HOUR, {{ date_expr }}) +{% endmacro %} diff --git a/macros/utils/cross_db_utils/hour_of_week.sql b/macros/utils/cross_db_utils/hour_of_week.sql index d26121179..c23c9fc53 100644 --- a/macros/utils/cross_db_utils/hour_of_week.sql +++ b/macros/utils/cross_db_utils/hour_of_week.sql @@ -38,3 +38,8 @@ {% macro trino__edr_hour_of_week_expression(date_expr) %} date_format({{ date_expr }}, '%W%H') {% endmacro %} + +{% macro fabric__edr_hour_of_week_expression(date_expr) %} + format({{ date_expr }}, 'ddddHH') +{% endmacro %} + diff --git a/macros/utils/cross_db_utils/stddev.sql b/macros/utils/cross_db_utils/stddev.sql new file mode 100644 index 000000000..0e7467013 --- /dev/null +++ b/macros/utils/cross_db_utils/stddev.sql @@ -0,0 +1,12 @@ +{# Standard deviation made to work on all adapters #} +{% macro edr_stddev(metric_value) -%} + {{ return(adapter.dispatch('edr_stddev', 'elementary') (metric_value)) }} +{%- endmacro %} + +{% macro default__edr_stddev(metric_value) %} + stddev(metric_value) +{% endmacro %} + +{% macro fabric__edr_stddev(metric_value) %} + stdev(metric_value) +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/cross_db_utils/target_database.sql b/macros/utils/cross_db_utils/target_database.sql index 73e407d2c..61f95fa3c 100644 --- a/macros/utils/cross_db_utils/target_database.sql +++ b/macros/utils/cross_db_utils/target_database.sql @@ -15,6 +15,10 @@ {% do return(target.database) %} {% endmacro %} +{% macro fabric__target_database() %} + {% do return(target.database) %} +{% endmacro %} + {% macro bigquery__target_database() %} {% do return(target.project) %} {% endmacro %} diff --git a/macros/utils/cross_db_utils/time_trunc.sql b/macros/utils/cross_db_utils/time_trunc.sql index 6d07d98fd..7503d0353 100644 --- a/macros/utils/cross_db_utils/time_trunc.sql +++ b/macros/utils/cross_db_utils/time_trunc.sql @@ -9,4 +9,8 @@ {% macro bigquery__edr_time_trunc(date_part, date_expression) %} timestamp_trunc(cast({{ date_expression }} as timestamp), {{ date_part }}) +{% endmacro %} + +{% macro fabric__edr_time_trunc(date_part, date_expression) %} + datetrunc({{date_part}}, cast({{ date_expression }} as {{ elementary.edr_type_timestamp() }})) {% endmacro %} \ No newline at end of file diff --git a/macros/utils/data_types/bool_type_values.sql b/macros/utils/data_types/bool_type_values.sql new file mode 100644 index 000000000..d3d412c09 --- /dev/null +++ b/macros/utils/data_types/bool_type_values.sql @@ -0,0 +1,48 @@ +{# Evaluate bool - For jinja bool true/false evaluation towards DBs compatible bool type#} + +{# Due to "faking" of bool using bit in T-SQL we have distinguish representation of true/false #} +{# We have different bools for use in parameter wheres vs. in columns as values#} + + +{% macro edr_evaluate_bool(return_value) %} +{{ return(adapter.dispatch('edr_bool', 'elementary')(return_value)) }} +{%endmacro%} + +{% macro edr_evaluate_bool_condition(return_value) %} +{{ return(adapter.dispatch('edr_bool_condition', 'elementary')(return_value)) }} +{%endmacro%} + +{% macro default__edr_bool(return_value) %} + {% if return_value == true %} + {%do return(true)%} + {% else %} + {%do return(false)%} + {% endif %} +{% endmacro %} + + +{% macro fabric__edr_bool(return_value) %} + {% if return_value == true %} + {%do return(1)%} + {% else %} + {%do return(0)%} + {% endif %} +{% endmacro %} + + +{% macro default__edr_bool_condition(return_value) %} + {% if return_value == true %} + {%do return(true)%} + {% else %} + {%do return(false)%} + {% endif %} +{% endmacro %} + + +{% macro fabric__edr_bool_condition(return_value) %} + {% if return_value == true %} + {%do return('1=1')%} + {% else %} + {%do return('0=1')%} + {% endif %} +{% endmacro %} diff --git a/macros/utils/data_types/data_type.sql b/macros/utils/data_types/data_type.sql index 8a0d1ed1a..781d905ec 100644 --- a/macros/utils/data_types/data_type.sql +++ b/macros/utils/data_types/data_type.sql @@ -19,6 +19,10 @@ {% do return("BOOL") %} {% endmacro %} +{% macro fabric__edr_type_bool() %} + {% do return("bit") %} +{% endmacro %} + {%- macro edr_type_string() -%} {{ return(adapter.dispatch('edr_type_string', 'elementary')()) }} @@ -59,6 +63,9 @@ {% do return("varchar") %} {% endmacro %} +{% macro fabric__edr_type_string() %} + {% do return("varchar(4096)") %} +{% endmacro %} @@ -152,3 +159,7 @@ {% macro trino__edr_type_timestamp() %} timestamp(6) {% endmacro %} + +{% macro fabric__edr_type_timestamp() %} + datetime2(2) +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/data_types/data_type_list.sql b/macros/utils/data_types/data_type_list.sql index 4625503e0..5e9cb64d6 100644 --- a/macros/utils/data_types/data_type_list.sql +++ b/macros/utils/data_types/data_type_list.sql @@ -131,3 +131,24 @@ {%- endif %} {% endmacro %} + +{% macro fabric__data_type_list(data_type) %} + + {% set string_list = ['varchar', 'char'] | list %} + {% set numeric_list = ['smallint', 'int', 'bigint', 'float', 'real', 'numeric'] | list %} + {% set timestamp_list = ['datetime2','date', 'time'] | list %} + {% set boolean_list = ["bit"] | list %} + + {%- if data_type == 'string' %} + {{ return(string_list) }} + {%- elif data_type == 'numeric' %} + {{ return(numeric_list) }} + {%- elif data_type == 'timestamp' %} + {{ return(timestamp_list) }} + {%- elif data_type == "boolean" %} + {{ return(boolean_list) }} + {%- else %} + {{ return([]) }} + {%- endif %} + +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/table_operations/create_or_replace.sql b/macros/utils/table_operations/create_or_replace.sql index 17cef9988..d3c5bcf26 100644 --- a/macros/utils/table_operations/create_or_replace.sql +++ b/macros/utils/table_operations/create_or_replace.sql @@ -35,3 +35,9 @@ {% do dbt.drop_relation_if_exists(relation) %} {% do elementary.run_query(dbt.create_table_as(temporary, relation, sql_query)) %} {% endmacro %} + +{% macro fabric__create_or_replace(temporary, relation, sql_query) %} + {% do dbt.drop_relation_if_exists(relation) %} + {% do elementary.run_query(dbt.create_table_as(temporary, relation, sql_query)) %} +{% endmacro %} + diff --git a/macros/utils/table_operations/get_relation_max_length.sql b/macros/utils/table_operations/get_relation_max_length.sql index 84b8ce6d4..7b561429e 100644 --- a/macros/utils/table_operations/get_relation_max_length.sql +++ b/macros/utils/table_operations/get_relation_max_length.sql @@ -30,3 +30,11 @@ {% macro trino__get_relation_max_name_length(temporary, relation, sql_query) %} {{ return(128) }} {% endmacro %} + +{% macro fabric__get_relation_max_name_length(temporary, relation, sql_query) %} + {{ return(64) }} +{% endmacro %} + +{% macro sqlserver__get_relation_max_name_length(temporary, relation, sql_query) %} + {{ return(64) }} +{% endmacro %} diff --git a/macros/utils/table_operations/has_temp_table_support.sql b/macros/utils/table_operations/has_temp_table_support.sql index e6db1ce55..54dc80875 100644 --- a/macros/utils/table_operations/has_temp_table_support.sql +++ b/macros/utils/table_operations/has_temp_table_support.sql @@ -18,3 +18,6 @@ {% do return(false) %} {% endmacro %} +{% macro fabric__has_temp_table_support() %} + {% do return(false) %} +{% endmacro %} diff --git a/macros/utils/table_operations/insert_as_select.sql b/macros/utils/table_operations/insert_as_select.sql index 2e30add23..384495ba0 100644 --- a/macros/utils/table_operations/insert_as_select.sql +++ b/macros/utils/table_operations/insert_as_select.sql @@ -2,6 +2,11 @@ {# when calling this macro, you need to add depends on ref comment #} {# ref_model and select_query need to have the same columns #} + {{ return(adapter.dispatch('insert_as_select', 'elementary')(table_relation, select_query)) }} + +{% endmacro %} + +{% macro default__insert_as_select(table_relation, select_query) %} {%- set insert_query %} insert into {{ table_relation }} with tmp_table as ( @@ -11,5 +16,15 @@ {%- endset %} {{ return(insert_query) }} +{% endmacro %} + +{% macro fabric__insert_as_select(table_relation, select_query) %} + {%- set insert_query %} + insert into {{ table_relation }} + select * from ( + {{ select_query }} + ) as tmp_table + {%- endset %} + {{ return(insert_query) }} {% endmacro %} \ No newline at end of file diff --git a/macros/utils/table_operations/insert_rows.sql b/macros/utils/table_operations/insert_rows.sql index 19826d762..a2091426e 100644 --- a/macros/utils/table_operations/insert_rows.sql +++ b/macros/utils/table_operations/insert_rows.sql @@ -157,6 +157,10 @@ {{- return(string_value | replace("'", "''")) -}} {%- endmacro -%} +{%- macro fabric__escape_special_chars(string_value) -%} + {{- return(string_value | replace("'", "''")) -}} +{%- endmacro -%} + {%- macro render_value(value, data_type) -%} {%- if value is defined and value is not none -%} {%- if value is number -%} diff --git a/models/edr/alerts/alerts_anomaly_detection.sql b/models/edr/alerts/alerts_anomaly_detection.sql index 2ef6e5b60..ad3afd3c3 100644 --- a/models/edr/alerts/alerts_anomaly_detection.sql +++ b/models/edr/alerts/alerts_anomaly_detection.sql @@ -34,7 +34,7 @@ alerts_anomaly_detection as ( status, result_rows from elementary_test_results - where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'anomaly_detection' + where {{ edr_evaluate_bool_condition ( not elementary.get_config_var('disable_test_alerts')) }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'anomaly_detection' ) select * from alerts_anomaly_detection diff --git a/models/edr/alerts/alerts_dbt_models.sql b/models/edr/alerts/alerts_dbt_models.sql index 7815ed110..834ffd460 100644 --- a/models/edr/alerts/alerts_dbt_models.sql +++ b/models/edr/alerts/alerts_dbt_models.sql @@ -76,4 +76,4 @@ select model_execution_id as alert_id, status, full_refresh from error_models -where {{ not elementary.get_config_var('disable_model_alerts') }} and lower(status) != 'success' {%- if elementary.get_config_var('disable_skipped_model_alerts') -%} and lower(status) != 'skipped' {%- endif -%} +where {{ edr_evaluate_bool_condition ( not elementary.get_config_var('disable_model_alerts')) }} and lower(status) != 'success' {%- if elementary.get_config_var('disable_skipped_model_alerts') -%} and lower(status) != 'skipped' {%- endif -%} diff --git a/models/edr/alerts/alerts_dbt_source_freshness.sql b/models/edr/alerts/alerts_dbt_source_freshness.sql index afbe27dc2..94f7caa8f 100644 --- a/models/edr/alerts/alerts_dbt_source_freshness.sql +++ b/models/edr/alerts/alerts_dbt_source_freshness.sql @@ -43,4 +43,4 @@ select results.filter as freshness_filter from results join sources on results.unique_id = sources.unique_id -where {{ not elementary.get_config_var('disable_source_freshness_alerts') }} and lower(status) != 'pass' +where {{ edr_evaluate_bool_condition ( not elementary.get_config_var('disable_source_freshness_alerts')) }} and lower(status) != 'pass' diff --git a/models/edr/alerts/alerts_dbt_tests.sql b/models/edr/alerts/alerts_dbt_tests.sql index e55b780aa..ec2d65a34 100644 --- a/models/edr/alerts/alerts_dbt_tests.sql +++ b/models/edr/alerts/alerts_dbt_tests.sql @@ -34,7 +34,11 @@ alerts_dbt_tests as ( status, result_rows from elementary_test_results - where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {% if elementary.get_config_var('disable_warn_alerts') %} and lower(status) != 'warn' {% endif %} {% if elementary.get_config_var('disable_skipped_test_alerts') %} and lower(status) != 'skipped' {% endif %} and test_type = 'dbt_test' + where {{ edr_evaluate_bool_condition (not elementary.get_config_var('disable_test_alerts')) }} + and lower(status) != 'pass' + {% if elementary.get_config_var('disable_warn_alerts') %} and lower(status) != 'warn' {% endif %} + {% if elementary.get_config_var('disable_skipped_test_alerts') %} and lower(status) != 'skipped' {% endif %} + and test_type = 'dbt_test' ) select * from alerts_dbt_tests diff --git a/models/edr/alerts/alerts_schema_changes.sql b/models/edr/alerts/alerts_schema_changes.sql index 5f39cb31b..0fcc2948e 100644 --- a/models/edr/alerts/alerts_schema_changes.sql +++ b/models/edr/alerts/alerts_schema_changes.sql @@ -35,7 +35,7 @@ alerts_schema_changes as ( status, result_rows from elementary_test_results - where {{ not elementary.get_config_var('disable_test_alerts') }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'schema_change' + where {{ edr_evaluate_bool_condition ( not elementary.get_config_var('disable_test_alerts')) }} and lower(status) != 'pass' {%- if elementary.get_config_var('disable_warn_alerts') -%} and lower(status) != 'warn' {%- endif -%} {%- if elementary.get_config_var('disable_skipped_test_alerts') -%} and lower(status) != 'skipped' {%- endif -%} and test_type = 'schema_change' ) select * from alerts_schema_changes diff --git a/models/edr/data_monitoring/anomaly_detection/anomaly_threshold_sensitivity.sql b/models/edr/data_monitoring/anomaly_detection/anomaly_threshold_sensitivity.sql index 656f8f5fd..0cd977bfd 100644 --- a/models/edr/data_monitoring/anomaly_detection/anomaly_threshold_sensitivity.sql +++ b/models/edr/data_monitoring/anomaly_detection/anomaly_threshold_sensitivity.sql @@ -21,13 +21,13 @@ score_sensitivity as ( training_avg as metric_avg, training_stddev as metric_stddev, anomaly_score, - case when abs(anomaly_score) >= 1.5 then true else false end as {{ elementary.edr_quote_column('is_anomaly_1_5') }}, - case when abs(anomaly_score) >= 2 then true else false end as {{ elementary.edr_quote_column('is_anomaly_2') }}, - case when abs(anomaly_score) >= 2.5 then true else false end as {{ elementary.edr_quote_column('is_anomaly_2_5') }}, - case when abs(anomaly_score) >= 3 then true else false end as {{ elementary.edr_quote_column('is_anomaly_3') }}, - case when abs(anomaly_score) >= 3.5 then true else false end as {{ elementary.edr_quote_column('is_anomaly_3_5') }}, - case when abs(anomaly_score) >= 4 then true else false end as {{ elementary.edr_quote_column('is_anomaly_4') }}, - case when abs(anomaly_score) >= 4.5 then true else false end as {{ elementary.edr_quote_column('is_anomaly_4_5') }} + case when abs(anomaly_score) >= 1.5 then {{ elementary.edr_evaluate_bool(true) }} else {{ elementary.edr_evaluate_bool(false) }} end as {{ elementary.edr_quote_column('is_anomaly_1_5') }}, + case when abs(anomaly_score) >= 2 then {{ elementary.edr_evaluate_bool(true) }} else {{ elementary.edr_evaluate_bool(false) }} end as {{ elementary.edr_quote_column('is_anomaly_2') }}, + case when abs(anomaly_score) >= 2.5 then {{ elementary.edr_evaluate_bool(true) }} else {{ elementary.edr_evaluate_bool(false) }} end as {{ elementary.edr_quote_column('is_anomaly_2_5') }}, + case when abs(anomaly_score) >= 3 then {{ elementary.edr_evaluate_bool(true)}} else {{ elementary.edr_evaluate_bool(false) }} end as {{ elementary.edr_quote_column('is_anomaly_3') }}, + case when abs(anomaly_score) >= 3.5 then {{ elementary.edr_evaluate_bool(true) }} else {{ elementary.edr_evaluate_bool(false) }} end as {{ elementary.edr_quote_column('is_anomaly_3_5') }}, + case when abs(anomaly_score) >= 4 then {{ elementary.edr_evaluate_bool(true)}} else {{ elementary.edr_evaluate_bool(false) }} end as {{ elementary.edr_quote_column('is_anomaly_4') }}, + case when abs(anomaly_score) >= 4.5 then {{ elementary.edr_evaluate_bool(true) }} else {{ elementary.edr_evaluate_bool(false) }} end as {{ elementary.edr_quote_column('is_anomaly_4_5') }} from metrics_anomaly_score where abs(anomaly_score) >= 1.5 diff --git a/models/edr/data_monitoring/anomaly_detection/metrics_anomaly_score.sql b/models/edr/data_monitoring/anomaly_detection/metrics_anomaly_score.sql index 0c8094bdc..9b18de5c2 100644 --- a/models/edr/data_monitoring/anomaly_detection/metrics_anomaly_score.sql +++ b/models/edr/data_monitoring/anomaly_detection/metrics_anomaly_score.sql @@ -27,12 +27,24 @@ time_window_aggregation as ( bucket_duration_hours, updated_at, avg(metric_value) over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) as training_avg, - stddev(metric_value) over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) as training_stddev, + {{ elementary.edr_stddev('metric_value') }} over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) as training_stddev, count(metric_value) over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) as training_set_size, last_value(bucket_end) over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) training_end, first_value(bucket_end) over (partition by metric_name, full_table_name, column_name order by bucket_start asc rows between unbounded preceding and current row) as training_start from data_monitoring_metrics - {{ dbt_utils.group_by(12) }} + group by + id, + full_table_name, + column_name, + dimension, + dimension_value, + metric_name, + metric_value, + source_value, + bucket_start, + bucket_end, + bucket_duration_hours, + updated_at ), metrics_anomaly_score as ( @@ -63,9 +75,21 @@ metrics_anomaly_score as ( metric_value is not null and training_avg is not null and bucket_end >= {{ elementary.edr_timeadd('day', '-7', elementary.edr_date_trunc('day', elementary.edr_current_timestamp())) }} - {{ dbt_utils.group_by(15) }} - order by bucket_end desc - + group by + id, + full_table_name, + column_name, + dimension, + dimension_value, + metric_name, + metric_value, + bucket_start, + bucket_end, + training_avg, + training_stddev, + training_start, + training_end, + training_set_size ), @@ -89,8 +113,10 @@ final as ( training_set_size, updated_at, case - when abs(anomaly_score) > {{ elementary.get_config_var('anomaly_sensitivity') }} then true - else false end + when abs(anomaly_score) > {{ elementary.get_config_var('anomaly_sensitivity') }} + then {{ elementary.edr_evaluate_bool(true) }} + else {{ elementary.edr_evaluate_bool(false) }} + end as is_anomaly from metrics_anomaly_score ) diff --git a/models/edr/dbt_artifacts/dbt_artifacts_hashes.sql b/models/edr/dbt_artifacts/dbt_artifacts_hashes.sql index 3cd1bfb24..8985f0189 100644 --- a/models/edr/dbt_artifacts/dbt_artifacts_hashes.sql +++ b/models/edr/dbt_artifacts/dbt_artifacts_hashes.sql @@ -23,4 +23,3 @@ select from {{ ref(artifact_model) }} {% if not loop.last %} union all {% endif %} {% endfor %} -order by metadata_hash diff --git a/models/edr/run_results/model_run_results.sql b/models/edr/run_results/model_run_results.sql index e30ade987..87d906159 100644 --- a/models/edr/run_results/model_run_results.sql +++ b/models/edr/run_results/model_run_results.sql @@ -42,12 +42,12 @@ SELECT models.alias, ROW_NUMBER() OVER (PARTITION BY run_results.unique_id ORDER BY run_results.generated_at DESC) AS model_invocation_reverse_index, CASE WHEN FIRST_VALUE(invocation_id) OVER (PARTITION BY {{ elementary.edr_time_trunc('day', 'run_results.generated_at') }} ORDER BY run_results.generated_at ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) = invocation_id - THEN TRUE - ELSE FALSE + THEN {{ elementary.edr_evaluate_bool(true) }} + ELSE {{ elementary.edr_evaluate_bool(false) }} END AS is_the_first_invocation_of_the_day, CASE WHEN LAST_VALUE(invocation_id) OVER (PARTITION BY {{ elementary.edr_time_trunc('day', 'run_results.generated_at') }} ORDER BY run_results.generated_at ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) = invocation_id - THEN TRUE - ELSE FALSE + THEN {{ elementary.edr_evaluate_bool(true) }} + ELSE {{ elementary.edr_evaluate_bool(false) }} END AS is_the_last_invocation_of_the_day FROM dbt_run_results run_results diff --git a/models/edr/system/monitors_runs.sql b/models/edr/system/monitors_runs.sql index cda28d65d..f20bb3c4b 100644 --- a/models/edr/system/monitors_runs.sql +++ b/models/edr/system/monitors_runs.sql @@ -20,7 +20,11 @@ max_bucket_end as ( max(bucket_end) as last_bucket_end, min(bucket_end) as first_bucket_end from data_monitoring_metrics - group by 1,2,3,4 + group by + full_table_name, + column_name, + metric_name, + metric_properties )