Skip to content

Commit 6129705

Browse files
DVAlexHiggsDVAlexHiggs
committed
Merge branch 'stage/0.7.9'
2 parents f009ed8 + fd00c61 commit 6129705

22 files changed

+866
-193
lines changed

‎dbt_project.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
name: dbtvault
2-
version: 0.7.8
3-
require-dbt-version: [">=0.20.0", "<0.22.0"]
2+
version: 0.7.9
3+
require-dbt-version: [">=1.0.0", "<2.0.0"]
44
config-version: 2
55

6-
source-paths: ["models"]
7-
analysis-paths: ["analysis"]
6+
model-paths: ["models"]
7+
analysis-paths: ["analyses"]
88
test-paths: ["tests"]
9-
data-paths: ["data"]
9+
seed-paths: ["seeds"]
1010
macro-paths: ["macros"]
1111
docs-paths: ["docs"]
1212

1313
target-path: "target"
1414
clean-targets:
1515
- "target"
16-
- "dbt_modules"
16+
- "dbt_packages"
1717

1818
vars:
1919
hash: MD5

‎macros/internal/multikey.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
{%- if prefix -%}
2020
{{- dbtvault.prefix([col], prefix[0], alias_target='target') }} {{ condition }} {{ dbtvault.prefix([col], prefix[1]) -}}
2121
{%- endif %}
22-
{%- if not loop.last %} {{ operator }} {% endif %}
22+
{%- if not loop.last %} {{ operator }} {% endif -%}
2323
{% endfor -%}
2424
{%- else -%}
2525
{%- if dbtvault.is_list(columns) -%}

‎macros/materialisations/incremental_bridge_helpers.sql

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{% macro is_bridge_incremental() %}
2+
{#-- do not run introspective queries in parsing #}
3+
{% if not execute %}
4+
{{ return(False) }}
5+
{% else %}
6+
{% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
7+
8+
{{ return(relation is not none
9+
and relation.type == 'table'
10+
and model.config.materialized == 'bridge_incremental'
11+
and not flags.FULL_REFRESH) }}
12+
{% endif %}
13+
{% endmacro %}
14+
15+
{% macro incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %}
16+
{%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
17+
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
18+
19+
TRUNCATE TABLE {{ target_relation }};
20+
21+
INSERT INTO {{ target_relation }} ({{ dest_cols_csv }})
22+
(
23+
SELECT {{ dest_cols_csv }}
24+
FROM {{ tmp_relation }}
25+
);
26+
{%- endmacro %}

‎macros/materialisations/incremental_bridge_materialization.sql

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{% materialization bridge_incremental, default -%}
2+
3+
{% set full_refresh_mode = flags.FULL_REFRESH %}
4+
5+
{% set target_relation = this %}
6+
{% set existing_relation = load_relation(this) %}
7+
{% set tmp_relation = make_temp_relation(this) %}
8+
9+
{{ run_hooks(pre_hooks, inside_transaction=False) }}
10+
11+
-- `BEGIN` happens here:
12+
{{ run_hooks(pre_hooks, inside_transaction=True) }}
13+
14+
{% set to_drop = [] %}
15+
{% if existing_relation is none %}
16+
{% set build_sql = create_table_as(False, target_relation, sql) %}
17+
{% elif existing_relation.is_view or full_refresh_mode %}
18+
{#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
19+
{% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
20+
{% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
21+
{% do adapter.drop_relation(backup_relation) %}
22+
23+
{% do adapter.rename_relation(target_relation, backup_relation) %}
24+
{% set build_sql = create_table_as(False, target_relation, sql) %}
25+
{% do to_drop.append(backup_relation) %}
26+
{% else %}
27+
28+
{% set tmp_relation = make_temp_relation(target_relation) %}
29+
{% do run_query(create_table_as(True, tmp_relation, sql)) %}
30+
{% do adapter.expand_target_column_types(
31+
from_relation=tmp_relation,
32+
to_relation=target_relation) %}
33+
{% set build_sql = dbtvault.incremental_bridge_replace(tmp_relation, target_relation) %}
34+
{% endif %}
35+
36+
{% call statement("main") %}
37+
{{ build_sql }}
38+
{% endcall %}
39+
40+
{{ run_hooks(post_hooks, inside_transaction=True) }}
41+
42+
-- `COMMIT` happens here
43+
{% do adapter.commit() %}
44+
45+
{% for rel in to_drop %}
46+
{% do adapter.drop_relation(rel) %}
47+
{% endfor %}
48+
49+
{{ run_hooks(post_hooks, inside_transaction=False) }}
50+
51+
{{ return({'relations': [target_relation]}) }}
52+
53+
{%- endmaterialization %}

‎macros/materialisations/incremental_pit_helper.sql

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{% macro is_pit_incremental() %}
2+
{#-- do not run introspective queries in parsing #}
3+
{% if not execute %}
4+
{{ return(False) }}
5+
{% else %}
6+
{% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
7+
8+
{{ return(relation is not none
9+
and relation.type == 'table'
10+
and model.config.materialized == 'pit_incremental'
11+
and not flags.FULL_REFRESH) }}
12+
{% endif %}
13+
{% endmacro %}
14+
15+
{% macro incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %}
16+
{%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
17+
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
18+
19+
TRUNCATE TABLE {{ target_relation }};
20+
21+
INSERT INTO {{ target_relation }} ({{ dest_cols_csv }})
22+
(
23+
SELECT {{ dest_cols_csv }}
24+
FROM {{ tmp_relation }}
25+
);
26+
{%- endmacro %}

‎macros/materialisations/incremental_pit_materialization.sql

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{% materialization pit_incremental, default -%}
2+
3+
{% set full_refresh_mode = flags.FULL_REFRESH %}
4+
5+
{% set target_relation = this %}
6+
{% set existing_relation = load_relation(this) %}
7+
{% set tmp_relation = make_temp_relation(this) %}
8+
9+
{{ run_hooks(pre_hooks, inside_transaction=False) }}
10+
11+
-- `BEGIN` happens here:
12+
{{ run_hooks(pre_hooks, inside_transaction=True) }}
13+
14+
{% set to_drop = [] %}
15+
{% if existing_relation is none %}
16+
{% set build_sql = create_table_as(False, target_relation, sql) %}
17+
{% elif existing_relation.is_view or full_refresh_mode %}
18+
{#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
19+
{% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
20+
{% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
21+
{% do adapter.drop_relation(backup_relation) %}
22+
23+
{% do adapter.rename_relation(target_relation, backup_relation) %}
24+
{% set build_sql = create_table_as(False, target_relation, sql) %}
25+
{% do to_drop.append(backup_relation) %}
26+
{% else %}
27+
28+
{% set tmp_relation = make_temp_relation(target_relation) %}
29+
{% do run_query(create_table_as(True, tmp_relation, sql)) %}
30+
{% do adapter.expand_target_column_types(
31+
from_relation=tmp_relation,
32+
to_relation=target_relation) %}
33+
{% set build_sql = dbtvault.incremental_pit_replace(tmp_relation, target_relation) %}
34+
{% endif %}
35+
36+
{% call statement("main") %}
37+
{{ build_sql }}
38+
{% endcall %}
39+
40+
{{ run_hooks(post_hooks, inside_transaction=True) }}
41+
42+
-- `COMMIT` happens here
43+
{% do adapter.commit() %}
44+
45+
{% for rel in to_drop %}
46+
{% do adapter.drop_relation(rel) %}
47+
{% endfor %}
48+
49+
{{ run_hooks(post_hooks, inside_transaction=False) }}
50+
51+
{{ return({'relations': [target_relation]}) }}
52+
53+
{%- endmaterialization %}

‎macros/materialisations/period_mat_helpers.sql

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
select {{ target_cols_csv }} from ({{ filtered_sql.sql }})
5656
{%- endmacro %}
5757

58+
5859
{#-- GET_PERIOD_BOUNDARIES #}
5960

6061
{%- macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}
@@ -100,7 +101,6 @@
100101
{% do return(period_boundaries) %}
101102
{%- endmacro %}
102103

103-
104104
{#-- GET_PERIOD_OF_LOAD #}
105105

106106
{%- macro get_period_of_load(period, offset, start_timestamp) -%}
@@ -127,7 +127,6 @@
127127
{% do return(period_of_load) %}
128128
{%- endmacro -%}
129129

130-
131130
{#-- OTHER MACROS #}
132131

133132
{% macro is_vault_insert_by_period() %}

‎macros/materialisations/rank_mat_helpers.sql

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
{% do return(filtered_sql) %}
2525
{% endmacro %}
2626

27-
2827
{#-- OTHER MACROS #}
2928

3029
{% macro get_min_max_ranks(rank_column, rank_source_models) %}

‎macros/materialisations/shared_helpers.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212

1313
{%- macro is_any_incremental() -%}
14-
{%- if dbtvault.is_vault_insert_by_period() or dbtvault.is_vault_insert_by_rank() or is_incremental() -%}
14+
{%- if dbtvault.is_vault_insert_by_period() or dbtvault.is_vault_insert_by_rank() or dbtvault.is_pit_incremental() or dbtvault.is_bridge_incremental() or is_incremental() -%}
1515
{%- do return(true) -%}
1616
{%- else -%}
1717
{%- do return(false) -%}

‎macros/materialisations/vault_insert_by_period_materialization.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@
7171
period_boundaries.start_timestamp,
7272
period_boundaries.stop_timestamp, i) %}
7373

74+
{# This call statement drops and then creates a temporary table #}
75+
{# but MSSQL will fail to drop any temporary table created by a previous loop iteration #}
76+
{# See MSSQL note and drop code below #}
7477
{% call statement() -%}
7578
{{ create_table_as(True, tmp_relation, tmp_table_sql) }}
7679
{%- endcall %}

‎macros/staging/derive_columns.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
{%- do column_list.append(column_str) -%}
2626
{%- endfor -%}
2727

28-
{%- set concat_string = "CONCAT_WS(" ~ "'||', " ~ column_list | join(", ") ~ ") AS " ~ col -%}
28+
{% set concat_string = "CONCAT_WS(" ~ "'||', " ~ column_list | join(", ") ~ ") AS " ~ col %}
2929

3030
{%- do der_columns.append(concat_string) -%}
3131
{%- set exclude_columns = exclude_columns + columns[col] -%}

‎macros/supporting/date_timestamp.sql

Lines changed: 0 additions & 27 deletions
This file was deleted.

0 commit comments

Comments
 (0)