Skip to content

Commit 0cb4a0d

Browse files
author
Alex Higgs
committed
Release 0.7.6-b1
THIS IS A BETA RELEASE Whilst we have extensively tested this to the best of our ability, we cannot guarantee comprehensive coverage. Please report any issues via GitHub, and feel free to ask us questions on slack. Thanks! New features: - PITs - Bridges
2 parents 6ac0c90 + fe6bcb0 commit 0cb4a0d

File tree

10 files changed

+711
-5
lines changed

10 files changed

+711
-5
lines changed

‎.circleci/config.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ jobs:
5656
integration:
5757
docker:
5858
- image: cimg/python:3.8.5
59-
parallelism: 15
59+
parallelism: 20
6060
steps:
6161
- build_test_env
6262
- run:

‎macros/materialisations/incremental_bridge_helpers.sql

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{% macro is_bridge_incremental() %}
2+
{#-- do not run introspective queries in parsing #}
3+
{% if not execute %}
4+
{{ return(False) }}
5+
{% else %}
6+
{% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
7+
8+
{{ return(relation is not none
9+
and relation.type == 'table'
10+
and model.config.materialized == 'bridge_incremental'
11+
and not flags.FULL_REFRESH) }}
12+
{% endif %}
13+
{% endmacro %}
14+
15+
{% macro incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %}
16+
{%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
17+
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
18+
19+
TRUNCATE TABLE {{ target_relation }};
20+
21+
INSERT INTO {{ target_relation }} ({{ dest_cols_csv }})
22+
(
23+
SELECT {{ dest_cols_csv }}
24+
FROM {{ tmp_relation }}
25+
);
26+
{%- endmacro %}

‎macros/materialisations/incremental_bridge_materialization.sql

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{% materialization bridge_incremental, default -%}
2+
3+
{% set full_refresh_mode = flags.FULL_REFRESH %}
4+
5+
{% set target_relation = this %}
6+
{% set existing_relation = load_relation(this) %}
7+
{% set tmp_relation = make_temp_relation(this) %}
8+
9+
{{ run_hooks(pre_hooks, inside_transaction=False) }}
10+
11+
-- `BEGIN` happens here:
12+
{{ run_hooks(pre_hooks, inside_transaction=True) }}
13+
14+
{% set to_drop = [] %}
15+
{% if existing_relation is none %}
16+
{% set build_sql = create_table_as(False, target_relation, sql) %}
17+
{% elif existing_relation.is_view or full_refresh_mode %}
18+
{#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
19+
{% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
20+
{% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
21+
{% do adapter.drop_relation(backup_relation) %}
22+
23+
{% do adapter.rename_relation(target_relation, backup_relation) %}
24+
{% set build_sql = create_table_as(False, target_relation, sql) %}
25+
{% do to_drop.append(backup_relation) %}
26+
{% else %}
27+
28+
{% set tmp_relation = make_temp_relation(target_relation) %}
29+
{% do run_query(create_table_as(True, tmp_relation, sql)) %}
30+
{% do adapter.expand_target_column_types(
31+
from_relation=tmp_relation,
32+
to_relation=target_relation) %}
33+
{% set build_sql = dbtvault.incremental_bridge_replace(tmp_relation, target_relation) %}
34+
{% endif %}
35+
36+
{% call statement("main") %}
37+
{{ build_sql }}
38+
{% endcall %}
39+
40+
{{ run_hooks(post_hooks, inside_transaction=True) }}
41+
42+
-- `COMMIT` happens here
43+
{% do adapter.commit() %}
44+
45+
{% for rel in to_drop %}
46+
{% do adapter.drop_relation(rel) %}
47+
{% endfor %}
48+
49+
{{ run_hooks(post_hooks, inside_transaction=False) }}
50+
51+
{{ return({'relations': [target_relation]}) }}
52+
53+
{%- endmaterialization %}

‎macros/materialisations/incremental_pit_helper.sql

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{% macro is_pit_incremental() %}
2+
{#-- do not run introspective queries in parsing #}
3+
{% if not execute %}
4+
{{ return(False) }}
5+
{% else %}
6+
{% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
7+
8+
{{ return(relation is not none
9+
and relation.type == 'table'
10+
and model.config.materialized == 'pit_incremental'
11+
and not flags.FULL_REFRESH) }}
12+
{% endif %}
13+
{% endmacro %}
14+
15+
{% macro incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %}
16+
{%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
17+
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
18+
19+
TRUNCATE TABLE {{ target_relation }};
20+
21+
INSERT INTO {{ target_relation }} ({{ dest_cols_csv }})
22+
(
23+
SELECT {{ dest_cols_csv }}
24+
FROM {{ tmp_relation }}
25+
);
26+
{%- endmacro %}

‎macros/materialisations/incremental_pit_materialization.sql

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{% materialization pit_incremental, default -%}
2+
3+
{% set full_refresh_mode = flags.FULL_REFRESH %}
4+
5+
{% set target_relation = this %}
6+
{% set existing_relation = load_relation(this) %}
7+
{% set tmp_relation = make_temp_relation(this) %}
8+
9+
{{ run_hooks(pre_hooks, inside_transaction=False) }}
10+
11+
-- `BEGIN` happens here:
12+
{{ run_hooks(pre_hooks, inside_transaction=True) }}
13+
14+
{% set to_drop = [] %}
15+
{% if existing_relation is none %}
16+
{% set build_sql = create_table_as(False, target_relation, sql) %}
17+
{% elif existing_relation.is_view or full_refresh_mode %}
18+
{#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
19+
{% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
20+
{% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
21+
{% do adapter.drop_relation(backup_relation) %}
22+
23+
{% do adapter.rename_relation(target_relation, backup_relation) %}
24+
{% set build_sql = create_table_as(False, target_relation, sql) %}
25+
{% do to_drop.append(backup_relation) %}
26+
{% else %}
27+
28+
{% set tmp_relation = make_temp_relation(target_relation) %}
29+
{% do run_query(create_table_as(True, tmp_relation, sql)) %}
30+
{% do adapter.expand_target_column_types(
31+
from_relation=tmp_relation,
32+
to_relation=target_relation) %}
33+
{% set build_sql = dbtvault.incremental_pit_replace(tmp_relation, target_relation) %}
34+
{% endif %}
35+
36+
{% call statement("main") %}
37+
{{ build_sql }}
38+
{% endcall %}
39+
40+
{{ run_hooks(post_hooks, inside_transaction=True) }}
41+
42+
-- `COMMIT` happens here
43+
{% do adapter.commit() %}
44+
45+
{% for rel in to_drop %}
46+
{% do adapter.drop_relation(rel) %}
47+
{% endfor %}
48+
49+
{{ run_hooks(post_hooks, inside_transaction=False) }}
50+
51+
{{ return({'relations': [target_relation]}) }}
52+
53+
{%- endmaterialization %}

‎macros/materialisations/shared_helpers.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212

1313
{%- macro is_any_incremental() -%}
14-
{%- if dbtvault.is_vault_insert_by_period() or dbtvault.is_vault_insert_by_rank() or is_incremental() -%}
14+
{%- if dbtvault.is_vault_insert_by_period() or dbtvault.is_vault_insert_by_rank() or dbtvault.is_pit_incremental() or dbtvault.is_bridge_incremental() or is_incremental() -%}
1515
{%- do return(true) -%}
1616
{%- else -%}
1717
{%- do return(false) -%}

0 commit comments

Comments
 (0)