Merge pull request #463 from cagov/snowflake-cost-by-account

Snowflake cost by account

Author: summer-mothwood

transform/dbt_project.yml

@@ -39,10 +39,31 @@ models:
       snowflake_cost_tracking:
         +schema: snowflake_cost_tracking
 
+        # These staging models are a little unusual for two reasons:
+        #
+        #   1. They are incremental
+        #   2. They do some very light aggregation
+        #
+        # We do this because the source views in the SNOWFLAKE meta-database
+        # have a retention time of one year, and don't have very strong
+        # uniqueness constraints for their data grain.  By making the models
+        # incremental we ensure that we retain data that is older than the retention
+        # time. By aggregating to the usage date (and table/warehouse, if applicable),
+        # we ensure that we can correctly merge in the incremental updates
+        # without resulting in duplicated rows.
+        +materialized: incremental
+
+        # Never do a full refresh so that we avoid overwriting any old data.
+        # Otherwise we risk losing data beyond the 1 year retention window
+        +full_refresh: false
+
     intermediate:
       +database: "{{ env_var('DBT_TRANSFORM_DB', 'TRANSFORM_DEV') }}"
       state_entities:
         +schema: state_entities
+      snowflake_cost_tracking:
+        +schema: snowflake_cost_tracking
+        +materialized: view
 
     marts:
       # All marts models as tables to avoid needing write access to TRANSFORM
@@ -51,3 +72,6 @@ models:
       +database: "{{ env_var('DBT_ANALYTICS_DB', 'ANALYTICS_DEV') }}"
       state_entities:
         +schema: state_entities
+      snowflake_cost_tracking:
+        +schema: snowflake_cost_tracking
+        +materialized: table

transform/models/intermediate/snowflake_cost_tracking/_snowflake_cost_tracking.yml

@@ -0,0 +1,67 @@
+version: 2
+
+models:
+  - name: int_automatic_clustering_history
+    description: |
+      Credits used by automatic clustering, aggregated to account and usage date.
+    columns:
+      - name: organization_name
+        description: Organization name
+      - name: account_name
+        description: Account name
+      - name: usage_date
+        description: The date on which the usage occurred.
+      - name: credits_used
+        description: The total credits used for automatic clustering
+
+  - name: int_materialized_view_refresh_history
+    description: |
+      Credits used by materialized view refreshes, aggregated to account and usage date.
+    columns:
+      - name: organization_name
+        description: Organization name
+      - name: account_name
+        description: Account name
+      - name: usage_date
+        description: The date on which the usage occurred.
+      - name: credits_used
+        description: The total credits used for materialized view refreshes
+
+  - name: int_pipe_usage_history
+    description: |
+      Credits used by pipes, aggregated to account and usage date.
+    columns:
+      - name: organization_name
+        description: Organization name
+      - name: account_name
+        description: Account name
+      - name: usage_date
+        description: The date on which the usage occurred.
+      - name: credits_used
+        description: The total credits used by pipes
+
+  - name: int_storage_daily_history
+    description: |
+      Credits used by storage, aggregated to account and usage date.
+    columns:
+      - name: organization_name
+        description: Organization name
+      - name: account_name
+        description: Account name
+      - name: usage_date
+        description: The date on which the usage occurred.
+      - name: credits_used
+        description: The total credits used by storage
+
+  - name: int_warehouse_metering_history
+    description: |
+      Credits used by compute warehouses, aggregated to account and usage date.
+    columns:
+      - name: organization_name
+        description: Organization name
+      - name: account_name
+        description: Account name
+      - name: usage_date
+        description: The date on which the usage occurred.
+      - name: credits_used
+        description: The total credits used by warehouses

transform/models/intermediate/snowflake_cost_tracking/int_automatic_clustering_history.sql

@@ -0,0 +1,15 @@
+with source as (
+    select * from {{ ref('stg_automatic_clustering_history') }}
+),
+
+usage_history as (
+    select
+        organization_name,
+        account_name,
+        usage_date,
+        sum(credits_used) as credits_used
+    from source
+    group by all
+)
+
+select * from usage_history

transform/models/intermediate/snowflake_cost_tracking/int_materialized_view_refresh_history.sql

@@ -0,0 +1,15 @@
+with source as (
+    select * from {{ ref('stg_materialized_view_refresh_history') }}
+),
+
+usage_history as (
+    select
+        organization_name,
+        account_name,
+        usage_date,
+        sum(credits_used) as credits_used
+    from source
+    group by all
+)
+
+select * from usage_history

transform/models/intermediate/snowflake_cost_tracking/int_pipe_usage_history.sql

@@ -0,0 +1,15 @@
+with source as (
+    select * from {{ ref('stg_pipe_usage_history') }}
+),
+
+usage_history as (
+    select
+        organization_name,
+        account_name,
+        usage_date,
+        sum(credits_used) as credits_used
+    from source
+    group by all
+)
+
+select * from usage_history

transform/models/intermediate/snowflake_cost_tracking/int_storage_daily_history.sql

@@ -0,0 +1,15 @@
+with source as (
+    select * from {{ ref('stg_storage_daily_history') }}
+),
+
+usage_history as (
+    select
+        organization_name,
+        account_name,
+        usage_date,
+        sum(credits_used) as credits_used
+    from source
+    group by all
+)
+
+select * from usage_history

transform/models/intermediate/snowflake_cost_tracking/int_warehouse_metering_history.sql

@@ -0,0 +1,15 @@
+with source as (
+    select * from {{ ref('stg_warehouse_metering_history') }}
+),
+
+usage_history as (
+    select
+        organization_name,
+        account_name,
+        usage_date,
+        sum(credits_used) as credits_used
+    from source
+    group by all
+)
+
+select * from usage_history

transform/models/marts/snowflake_cost_tracking/_snowflake_cost_tracking.yml

@@ -0,0 +1,31 @@
+version: 2
+
+models:
+  - name: snowflake_costs_by_date.sql
+    description: |
+      Snowflake costs by date for the following:
+
+        * Automatic clustering
+        * Materialized view refreshes
+        * Pipe usage
+        * Storage
+        * Warehouse usage
+
+      Data are in long form, where `usage_type` indicates which
+      type of usage is measured in credits.
+    columns:
+      - name: account_name
+        description: Account name
+      - name: usage_date
+        description: Date on which the usage occurred
+      - name: usage_type
+        description: |
+          One of the following usage types:
+
+          * `'automatic clustering'`
+          * `'materialized view'`
+          * `'pipe'`
+          * `'storage'`
+          * `'warehouse'`
+      - name: credits_used
+        description: The credits used for the usage type and date.

transform/models/marts/snowflake_cost_tracking/snowflake_costs_by_date.sql

@@ -0,0 +1,68 @@
+/*
+TODO: this does not yet account for credits consumed by:
+
+  * Query acceleration
+  * Search optimization
+  * Replication/failover groups
+*/
+
+with automatic_clustering_history as (
+    select
+        account_name,
+        usage_date,
+        'automatic clustering' as usage_type,
+        credits_used
+    from {{ ref('int_automatic_clustering_history') }}
+),
+
+materialized_view_refresh_history as (
+    select
+        account_name,
+        usage_date,
+        'materialized view' as usage_type,
+        credits_used
+    from {{ ref('int_materialized_view_refresh_history') }}
+),
+
+pipe_usage_history as (
+    select
+        account_name,
+        usage_date,
+        'pipe' as usage_type,
+        credits_used
+    from {{ ref('int_pipe_usage_history') }}
+),
+
+storage_daily_history as (
+    select
+        account_name,
+        usage_date,
+        'storage' as usage_type,
+        credits_used
+    from {{ ref('int_storage_daily_history') }}
+),
+
+warehouse_metering_history as (
+    select
+        account_name,
+        usage_date,
+        'warehouse' as usage_type,
+        credits_used
+    from {{ ref('int_warehouse_metering_history') }}
+),
+
+-- Combine the data in long form to allow for easy
+-- aggregations and visualizations.
+combined as (
+    select * from automatic_clustering_history
+    union all
+    select * from materialized_view_refresh_history
+    union all
+    select * from pipe_usage_history
+    union all
+    select * from storage_daily_history
+    union all
+    select * from warehouse_metering_history
+)
+
+select * from combined