Skip to content

Commit ca533ea

Browse files
DVAlexHiggsDVAlexHiggs
committed
dbtvault 0.8.2
2 parents fb218f1 + 1a8f99b commit ca533ea

21 files changed

+730
-16
lines changed

‎dbt_project.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: dbtvault
2-
version: 0.8.1
2+
version: 0.8.2
33
require-dbt-version: [">=1.0.0", "<2.0.0"]
44
config-version: 2
55

‎macros/materialisations/incremental_bridge_materialization.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,4 @@
5454

5555
{{ return({'relations': [target_relation]}) }}
5656

57-
{%- endmaterialization -%}
57+
{%- endmaterialization -%}

‎macros/materialisations/period_mat_helpers/get_period_boundaries.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@
6464
{%- endset %}
6565

6666

67-
{% set period_boundaries_dict = dbt_utils.get_query_results_as_dict(period_boundary_sql) %}
67+
{% set period_boundaries_dict = dbtvault.get_query_results_as_dict(period_boundary_sql) %}
6868

6969
{% set period_boundaries = {'start_timestamp': period_boundaries_dict['START_TIMESTAMP'][0] | string,
7070
'stop_timestamp': period_boundaries_dict['STOP_TIMESTAMP'][0] | string,

‎macros/materialisations/period_mat_helpers/get_period_of_load.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
SELECT DATE_TRUNC(DATE_ADD( DATE('{{start_timestamp}}'), INTERVAL {{ offset }} {{ period }}), {{ period }} ) AS PERIOD_OF_LOAD
3434
{%- endset %}
3535

36-
{% set period_of_load_dict = dbt_utils.get_query_results_as_dict(period_of_load_sql) %}
36+
{% set period_of_load_dict = dbtvault.get_query_results_as_dict(period_of_load_sql) %}
3737

3838
{% set period_of_load = period_of_load_dict['PERIOD_OF_LOAD'][0] | string %}
3939

‎macros/materialisations/vault_insert_by_period_materialization.sql

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,15 @@
113113
period_of_load, rows_inserted,
114114
model.unique_id)) }}
115115

116+
{% if target.type == "sqlserver" %}
117+
{# In MSSQL a temporary table can only be dropped by the connection or session that created it #}
118+
{# so drop it now before the commit below closes this session #}
119+
{%- set drop_query_name = 'DROP_QUERY-' ~ i -%}
120+
{% call statement(drop_query_name, fetch_result=True) -%}
121+
DROP TABLE {{ tmp_relation }};
122+
{%- endcall %}
123+
{% endif %}
124+
116125
{% do to_drop.append(tmp_relation) %}
117126
{% do adapter.commit() %}
118127

‎macros/materialisations/vault_insert_by_rank_materialization.sql

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,15 @@
9393
rows_inserted,
9494
model.unique_id)) }}
9595

96+
{% if target.type == "sqlserver" %}
97+
{# In MSSQL a temporary table can only be dropped by the connection or session that created it #}
98+
{# so drop it now before the commit below closes this session #}
99+
{%- set drop_query_name = 'DROP_QUERY-' ~ i -%}
100+
{% call statement(drop_query_name, fetch_result=True) -%}
101+
DROP TABLE {{ tmp_relation }};
102+
{%- endcall %}
103+
{% endif %}
104+
96105
{% do to_drop.append(tmp_relation) %}
97106
{% do adapter.commit() %}
98107

‎macros/supporting/get_query_results_as_dict.sql

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{% macro get_query_results_as_dict(query) %}
2+
{{ return(adapter.dispatch('get_query_results_as_dict', 'dbtvault')(query)) }}
3+
{% endmacro %}
4+
5+
{% macro default__get_query_results_as_dict(query) %}
6+
{{ return(dbt_utils.get_query_results_as_dict(query)) }}
7+
{% endmacro %}
8+
9+
{% macro sqlserver__get_query_results_as_dict(query) %}
10+
11+
{%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%}
12+
13+
{{ query }}
14+
15+
{%- endcall -%}
16+
17+
{% set sql_results={} %}
18+
19+
{%- if execute -%}
20+
{% set sql_results_table = load_result('get_query_results').table.columns %}
21+
{% for column_name, column in sql_results_table.items() %}
22+
{# Column names in upper case for consistency #}
23+
{% do sql_results.update({column_name.upper(): column.values()}) %}
24+
{% endfor %}
25+
{%- endif -%}
26+
27+
{{ return(sql_results) }}
28+
29+
{% endmacro %}

‎macros/supporting/max_datetime.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@
1818

1919
{%- macro bigquery__max_datetime() %}
2020

21-
{% do return('9999-12-31 23:59:59.999') %}
21+
{% do return('9999-12-31 23:59:59.999999') %}
2222

2323
{% endmacro -%}

‎macros/tables/bigquery/eff_sat.sql

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
{%- macro bigquery__eff_sat(src_pk, src_dfk, src_sfk, src_start_date, src_end_date, src_eff, src_ldts, src_source, source_model) -%}
2+
3+
{{- dbtvault.check_required_parameters(src_pk=src_pk, src_dfk=src_dfk, src_sfk=src_sfk,
4+
src_start_date=src_start_date, src_end_date=src_end_date,
5+
src_eff=src_eff, src_ldts=src_ldts, src_source=src_source,
6+
source_model=source_model) -}}
7+
8+
{%- set src_pk = dbtvault.escape_column_names(src_pk) -%}
9+
{%- set src_dfk = dbtvault.escape_column_names(src_dfk) -%}
10+
{%- set src_sfk = dbtvault.escape_column_names(src_sfk) -%}
11+
{%- set src_start_date = dbtvault.escape_column_names(src_start_date) -%}
12+
{%- set src_end_date = dbtvault.escape_column_names(src_end_date) -%}
13+
{%- set src_eff = dbtvault.escape_column_names(src_eff) -%}
14+
{%- set src_ldts = dbtvault.escape_column_names(src_ldts) -%}
15+
{%- set src_source = dbtvault.escape_column_names(src_source) -%}
16+
17+
{%- set source_cols = dbtvault.expand_column_list(columns=[src_pk, src_dfk, src_sfk, src_start_date, src_end_date, src_eff, src_ldts, src_source]) -%}
18+
{%- set fk_cols = dbtvault.expand_column_list(columns=[src_dfk, src_sfk]) -%}
19+
{%- set dfk_cols = dbtvault.expand_column_list(columns=[src_dfk]) -%}
20+
{%- set is_auto_end_dating = config.get('is_auto_end_dating', default=false) %}
21+
22+
{%- set max_datetime = var('max_datetime', '9999-12-31 23:59:59.999999') %}
23+
24+
{{- dbtvault.prepend_generated_by() }}
25+
26+
WITH source_data AS (
27+
SELECT {{ dbtvault.prefix(source_cols, 'a', alias_target='source') }}
28+
FROM {{ ref(source_model) }} AS a
29+
WHERE {{ dbtvault.multikey(src_dfk, prefix='a', condition='IS NOT NULL') }}
30+
AND {{ dbtvault.multikey(src_sfk, prefix='a', condition='IS NOT NULL') }}
31+
{%- if model.config.materialized == 'vault_insert_by_period' %}
32+
AND __PERIOD_FILTER__
33+
{%- elif model.config.materialized == 'vault_insert_by_rank' %}
34+
AND __RANK_FILTER__
35+
{%- endif %}
36+
),
37+
38+
{%- if dbtvault.is_any_incremental() %}
39+
40+
{# Selecting the most recent records for each link hashkey -#}
41+
latest_records_unranked AS (
42+
SELECT {{ dbtvault.alias_all(source_cols, 'b') }},
43+
ROW_NUMBER() OVER (
44+
PARTITION BY {{ dbtvault.prefix([src_pk], 'b') }}
45+
ORDER BY b.{{ src_ldts }} DESC
46+
) AS row_num
47+
FROM {{ this }} AS b
48+
),
49+
50+
latest_records AS (
51+
SELECT *
52+
FROM latest_records_unranked
53+
WHERE row_num = 1
54+
),
55+
56+
{# Selecting the open records of the most recent records for each link hashkey -#}
57+
latest_open AS (
58+
SELECT {{ dbtvault.alias_all(source_cols, 'c') }}
59+
FROM latest_records AS c
60+
WHERE DATE(c.{{ src_end_date }}) = CAST(PARSE_DATETIME('%F %H:%M:%E6S', '{{ max_datetime }}') AS DATE)
61+
),
62+
63+
{# Selecting the closed records of the most recent records for each link hashkey -#}
64+
latest_closed AS (
65+
SELECT {{ dbtvault.alias_all(source_cols, 'd') }}
66+
FROM latest_records AS d
67+
WHERE DATE(d.{{ src_end_date }}) != CAST(PARSE_DATETIME('%F %H:%M:%E6S', '{{ max_datetime }}') AS DATE)
68+
),
69+
70+
{# Identifying the completely new link relationships to be opened in eff sat -#}
71+
new_open_records AS (
72+
SELECT DISTINCT
73+
{{ dbtvault.alias_all(source_cols, 'f') }}
74+
FROM source_data AS f
75+
LEFT JOIN latest_records AS lr
76+
ON {{ dbtvault.multikey(src_pk, prefix=['f','lr'], condition='=') }}
77+
WHERE {{ dbtvault.multikey(src_pk, prefix='lr', condition='IS NULL') }}
78+
),
79+
80+
{# Identifying the currently closed link relationships to be reopened in eff sat -#}
81+
new_reopened_records AS (
82+
SELECT DISTINCT
83+
{{ dbtvault.prefix([src_pk], 'lc') }},
84+
{{ dbtvault.alias_all(fk_cols, 'lc') }},
85+
lc.{{ src_start_date }} AS {{ src_start_date }},
86+
g.{{ src_end_date }} AS {{ src_end_date }},
87+
g.{{ src_eff }} AS {{ src_eff }},
88+
g.{{ src_ldts }},
89+
g.{{ src_source }}
90+
FROM source_data AS g
91+
INNER JOIN latest_closed AS lc
92+
ON {{ dbtvault.multikey(src_pk, prefix=['g','lc'], condition='=') }}
93+
WHERE CAST((g.{{ src_end_date }}) AS DATE) = CAST(PARSE_DATETIME('%F %H:%M:%E6S', '{{ max_datetime }}') AS DATE)
94+
),
95+
96+
{%- if is_auto_end_dating %}
97+
98+
{# Creating the closing records -#}
99+
{# Identifying the currently open relationships that need to be closed due to change in SFK(s) -#}
100+
new_closed_records AS (
101+
SELECT DISTINCT
102+
{{ dbtvault.prefix([src_pk], 'lo') }},
103+
{{ dbtvault.alias_all(fk_cols, 'lo') }},
104+
lo.{{ src_start_date }} AS {{ src_start_date }},
105+
h.{{ src_eff }} AS {{ src_end_date }},
106+
h.{{ src_eff }} AS {{ src_eff }},
107+
h.{{ src_ldts }},
108+
lo.{{ src_source }}
109+
FROM source_data AS h
110+
INNER JOIN latest_open AS lo
111+
ON {{ dbtvault.multikey(src_dfk, prefix=['lo', 'h'], condition='=') }}
112+
WHERE ({{ dbtvault.multikey(src_sfk, prefix=['lo', 'h'], condition='<>', operator='OR') }})
113+
),
114+
115+
{#- else if is_auto_end_dating -#}
116+
{% else %}
117+
118+
new_closed_records AS (
119+
SELECT DISTINCT
120+
lo.{{ src_pk }},
121+
{{ dbtvault.alias_all(fk_cols, 'lo') }},
122+
lo.{{ src_start_date }} AS {{ src_start_date }},
123+
h.{{ src_eff }} AS {{ src_end_date }},
124+
h.{{ src_eff }} AS {{ src_eff }},
125+
h.{{ src_ldts }},
126+
lo.{{ src_source }}
127+
FROM source_data AS h
128+
LEFT JOIN Latest_open AS lo
129+
ON lo.{{ src_pk }} = h.{{ src_pk }}
130+
LEFT JOIN latest_closed AS lc
131+
ON lc.{{ src_pk }} = h.{{ src_pk }}
132+
WHERE CAST((h.{{ src_end_date }}) AS DATE) != CAST(PARSE_DATETIME('%F %H:%M:%E6S', '{{ max_datetime }}') AS DATE)
133+
AND lo.{{ src_pk }} IS NOT NULL
134+
AND lc.{{ src_pk }} IS NULL
135+
),
136+
137+
{#- end if is_auto_end_dating -#}
138+
{%- endif %}
139+
140+
records_to_insert AS (
141+
SELECT * FROM new_open_records
142+
UNION DISTINCT
143+
SELECT * FROM new_reopened_records
144+
UNION DISTINCT
145+
SELECT * FROM new_closed_records
146+
)
147+
148+
{#- else if not dbtvault.is_any_incremental() -#}
149+
{%- else %}
150+
151+
records_to_insert AS (
152+
SELECT {{ dbtvault.alias_all(source_cols, 'i') }}
153+
FROM source_data AS i
154+
)
155+
156+
{#- end if not dbtvault.is_any_incremental() -#}
157+
{%- endif %}
158+
159+
SELECT *
160+
FROM records_to_insert
161+
162+
{%- endmacro -%}

0 commit comments

Comments
 (0)