From 546e197ae1bc0f2a02561cdd2bbcad34db1e1fec Mon Sep 17 00:00:00 2001 From: roman Date: Thu, 17 Oct 2024 16:55:49 +0200 Subject: [PATCH] bigquery demo monitors part 1 --- monitors.yaml | 320 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 320 insertions(+) diff --git a/monitors.yaml b/monitors.yaml index 8dd1d2a2..6844e581 100644 --- a/monitors.yaml +++ b/monitors.yaml @@ -960,3 +960,323 @@ monitors: different_rows_count: 10 different_rows_percent: 10 + + + + + + + + + + + + + + + + +# #################################################### +# OOO -> BIGQUERY +# #################################################### + +# Redshift<>Bigquery XDB DIFF monitors + + xdb_diff__redshift_bigquery__egress_10M_: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 8824 + table: demo.diff.transactions_v1_10m + dataset_b: + connection_id: 8377 + table: datafold-demo-429713.diff.transactions_v2_10m + primary_key: + - transaction_id + columns_to_compare: + - amount + - amount_base + - created_at + - discount + - first_name + - last_name + - number_of_attempts + - number_of_products + - score + - updated_at + egress_limit: 10000000 + alert: + different_rows_count: 10 + different_rows_percent: 10 + + + xdb_diff__redshift_bigquery__egress_10M__sampling_0_001: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 8824 + table: demo.diff.transactions_v1_10m + dataset_b: + connection_id: 8377 + table: datafold-demo-429713.diff.transactions_v2_10m + primary_key: + - transaction_id + columns_to_compare: + - amount + - amount_base + - created_at + - discount + - first_name + - last_name + - number_of_attempts + - number_of_products + - score + - updated_at + egress_limit: 10000000 + sampling: + rate: 0.001 + alert: + different_rows_count: 10 + different_rows_percent: 10 + + + +# Microsoft SQL Server<>Bigquery XDB DIFF monitors + + xdb_diff__mssqlserver_bigquery__egress_10M_: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 8182 + table: demo.demo.transactions_v1_10m + dataset_b: + connection_id: 8377 + table: datafold-demo-429713.diff.transactions_v2_10m + primary_key: + - transaction_id + columns_to_compare: + - amount + - amount_base + - created_at + - discount + - first_name + - last_name + - number_of_attempts + - number_of_products + - score + - updated_at + egress_limit: 10000000 + alert: + different_rows_count: 10 + different_rows_percent: 10 + + + xdb_diff__mssqlserver_bigquery__egress_10M__sampling_0_001: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 8182 + table: demo.demo.transactions_v1_10m + dataset_b: + connection_id: 8377 + table: datafold-demo-429713.diff.transactions_v2_10m + primary_key: + - transaction_id + columns_to_compare: + - amount + - amount_base + - created_at + - discount + - first_name + - last_name + - number_of_attempts + - number_of_products + - score + - updated_at + egress_limit: 10000000 + sampling: + rate: 0.001 + alert: + different_rows_count: 10 + different_rows_percent: 10 + + + +# Databricks<>Bigquery XDB DIFF monitors + + xdb_diff__databricks_bigquery__egress_10M_: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 8357 + table: demo.diff.transactions_v1_10m + dataset_b: + connection_id: 8377 + table: datafold-demo-429713.diff.transactions_v2_10m + primary_key: + - transaction_id + columns_to_compare: + - amount + - amount_base + - created_at + - discount + - first_name + - last_name + - number_of_attempts + - number_of_products + - score + - updated_at + egress_limit: 10000000 + alert: + different_rows_count: 10 + different_rows_percent: 10 + + + xdb_diff__databricks_bigquery__egress_10M__sampling_0_001: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 8357 + table: demo.diff.transactions_v1_10m + dataset_b: + connection_id: 8377 + table: datafold-demo-429713.diff.transactions_v2_10m + primary_key: + - transaction_id + columns_to_compare: + - amount + - amount_base + - created_at + - discount + - first_name + - last_name + - number_of_attempts + - number_of_products + - score + - updated_at + egress_limit: 10000000 + sampling: + rate: 0.001 + alert: + different_rows_count: 10 + different_rows_percent: 10 + + + +# Snowflake<>Bigquery XDB DIFF monitors + + xdb_diff__snowflake_bigquery__egress_10M_: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 4932 + table: DEMO.DIFF.TRANSACTIONS_V1_10M + dataset_b: + connection_id: 8377 + table: datafold-demo-429713.diff.transactions_v2_10m + primary_key: + - transaction_id + columns_to_compare: + - amount + - amount_base + - created_at + - discount + - first_name + - last_name + - number_of_attempts + - number_of_products + - score + - updated_at + egress_limit: 10000000 + alert: + different_rows_count: 10 + different_rows_percent: 10 + + + xdb_diff__snowflake_bigquery__egress_10M__sampling_0_001: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 4932 + table: DEMO.DIFF.TRANSACTIONS_V1_10M + dataset_b: + connection_id: 8377 + table: datafold-demo-429713.diff.transactions_v2_10m + primary_key: + - transaction_id + columns_to_compare: + - amount + - amount_base + - created_at + - discount + - first_name + - last_name + - number_of_attempts + - number_of_products + - score + - updated_at + egress_limit: 10000000 + sampling: + rate: 0.001 + alert: + different_rows_count: 10 + different_rows_percent: 10 + + +# Bigquery<>Bigquery XDB DIFF monitors + + xdb_diff__bigquery_bigquery__egress_10M_: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 9171 + table: datafold-demo-429713.diff.transactions_v1_10m + dataset_b: + connection_id: 8377 + table: datafold-demo-429713.diff.transactions_v2_10m + primary_key: + - transaction_id + egress_limit: 10000000 + alert: + different_rows_count: 10 + different_rows_percent: 10 + + + xdb_diff__bigquery_bigquery__egress_10M__sampling_0_001: + type: diff + schedule: + cron: 15 15 * * * + datadiff: + dataset_a: + connection_id: 9171 + table: datafold-demo-429713.diff.transactions_v1_10m + dataset_b: + connection_id: 8377 + table: datafold-demo-429713.diff.transactions_v2_10m + primary_key: + - transaction_id + egress_limit: 10000000 + sampling: + rate: 0.001 + alert: + different_rows_count: 10 + different_rows_percent: 10 +