Skip to content

Commit 82c61bf

Browse files
DVAlexHiggsDVAlexHiggs
committed
Release 0.9.7
1 parent 3db7cc2 commit 82c61bf

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+998
-714
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,4 +89,4 @@ Talk to our developers and other members of our growing community, get support a
8989
[View our contribution guidelines](CONTRIBUTING.md)
9090

9191
## License
92-
[Apache 2.0](LICENSE.md)
92+
[Apache 2.0](LICENSE)

‎dbt_project.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: automate_dv
2-
version: 0.9.5
2+
version: 0.9.7
33
require-dbt-version: [">=1.0.0", "<2.0.0"]
44
config-version: 2
55

‎macros/internal/helpers/dateadd.sql

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
{% macro sqlserver__dateadd(datepart, interval, from_date_or_timestamp) %}
1919

2020
dateadd(
21-
millisecond,
22-
86399999,
21+
{{ datepart }},
22+
{{ interval }},
2323
CAST({{ from_date_or_timestamp }} AS DATETIME2)
2424
)
2525

‎macros/internal/helpers/stage_processing_macros/extract_column_names.sql

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
* This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
44
*/
55

6-
AutomateDV (f.k.a automate_dv)
7-
86
{%- macro extract_column_names(columns_dict=none) -%}
97

108
{%- set extracted_column_names = [] -%}

‎macros/internal/helpers/stage_processing_macros/extract_null_column_names.sql

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
* This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
44
*/
55

6-
AutomateDV (f.k.a automate_dv)
7-
86
{%- macro extract_null_column_names(columns_dict=none) -%}
97

108
{%- set extracted_column_names = [] -%}

‎macros/internal/helpers/stage_processing_macros/print_list.sql

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
* This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
44
*/
55

6-
AutomateDV (f.k.a automate_dv)
7-
86
{%- macro print_list(list_to_print=none, indent=4, columns_to_escape=none) -%}
97

108
{%- for col_name in list_to_print -%}

‎macros/internal/helpers/stage_processing_macros/process_columns_to_escape.sql

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
* This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
44
*/
55

6-
AutomateDV (f.k.a automate_dv)
7-
86
{%- macro process_columns_to_escape(derived_columns_list=none) -%}
97

108
{%- if derived_columns_list -%}

‎macros/internal/helpers/stage_processing_macros/process_columns_to_select.sql

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
* This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
44
*/
55

6-
AutomateDV (f.k.a automate_dv)
7-
86
{%- macro process_columns_to_select(columns_list=none, exclude_columns_list=none) -%}
97

108
{% set columns_list = columns_list | list %}

‎macros/internal/helpers/stage_processing_macros/process_hash_column_excludes.sql

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
* This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
44
*/
55

6-
AutomateDV (f.k.a automate_dv)
7-
86
{%- macro process_hash_column_excludes(hash_columns=none, source_columns=none) -%}
97

108
{%- set processed_hash_columns = {} -%}

‎macros/internal/helpers/timestamp_add.sql

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright (c) Business Thinking Ltd. 2019-2023
3+
* This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
4+
*/
5+
6+
{% macro timestamp_add(datepart, interval, from_date_or_timestamp) %}
7+
{{ return(adapter.dispatch('timestamp_add', 'automate_dv')(datepart=datepart, interval=interval,
8+
from_date_or_timestamp=from_date_or_timestamp)) }}
9+
{%- endmacro -%}
10+
11+
{%- macro default__timestamp_add(datepart, interval, from_date_or_timestamp) -%}
12+
13+
{%- if datepart is in ['day', 'week', 'month', 'quarter', 'year'] -%}
14+
{{ automate_dv.dateadd('millisecond', 86399999, from_date_or_timestamp) }}
15+
{%- elif datepart == 'microsecond' -%}
16+
{{ automate_dv.dateadd('microsecond', 1, from_date_or_timestamp) }}
17+
{%- elif datepart == 'millisecond' -%}
18+
{{ automate_dv.dateadd('microsecond', 999, from_date_or_timestamp) }}
19+
{%- elif datepart == 'second' -%}
20+
{{ automate_dv.dateadd('millisecond', 999, from_date_or_timestamp) }}
21+
{%- elif datepart == 'minute' -%}
22+
{{ automate_dv.dateadd('millisecond', 5999, from_date_or_timestamp) }}
23+
{%- elif datepart == 'hour' -%}
24+
{{ automate_dv.dateadd('millisecond', 3599999, from_date_or_timestamp) }}
25+
{%- endif -%}
26+
27+
{%- endmacro -%}
28+
29+
{% macro bigquery__timestamp_add(datepart, interval, from_date_or_timestamp) %}
30+
31+
{%- if datepart is in ['day', 'week', 'month', 'quarter', 'year'] -%}
32+
{{ automate_dv.dateadd('millisecond', 86399999, from_date_or_timestamp) }}
33+
{%- elif datepart == 'microsecond' -%}
34+
TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 1 microsecond)
35+
{%- elif datepart == 'millisecond' -%}
36+
TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 999 microsecond)
37+
{%- elif datepart == 'second' -%}
38+
TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 999 millisecond)
39+
{%- elif datepart == 'minute' -%}
40+
TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 5999 millisecond)
41+
{%- elif datepart == 'hour' -%}
42+
TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 3599999 millisecond)
43+
{%- endif -%}
44+
45+
{% endmacro %}
46+
47+

‎macros/internal/metadata_processing/get_escape_characters.sql

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,30 @@
55

66
{% macro get_escape_characters() -%}
77

8-
{%- set escape_char_left, escape_char_right = adapter.dispatch('get_escape_characters', 'automate_dv')() -%}
8+
{%- set default_escape_char_left, default_escape_char_right = adapter.dispatch('get_escape_characters', 'automate_dv')() -%}
99

10-
{%- do return((var('escape_char_left', escape_char_left), var('escape_char_right', escape_char_right))) -%}
10+
{%- if ((var('escape_char_left', default_escape_char_left) == '') and (var('escape_char_right', default_escape_char_right) == '')) -%}
11+
{%- set warning_message = 'Invalid escape_char_left and escape_char_right value provided. ' +
12+
'Using platform defaults ({}{})'.format(default_escape_char_left, default_escape_char_right) -%}
13+
{%- set escape_chars = (default_escape_char_left, default_escape_char_right) -%}
14+
15+
{%- elif var('escape_char_left', default_escape_char_left) == '' -%}
16+
{%- set warning_message = 'Invalid escape_char_left value provided. Using platform default ({})'.format(default_escape_char_left) -%}
17+
{%- set escape_chars = (default_escape_char_left, var('escape_char_right', default_escape_char_right)) -%}
18+
19+
{%- elif var('escape_char_right', default_escape_char_right) == '' -%}
20+
{%- set warning_message = 'Invalid escape_char_right value provided. Using platform default ({})'.format(default_escape_char_right) -%}
21+
{%- set escape_chars = (var('escape_char_left', default_escape_char_left), default_escape_char_right) -%}
22+
23+
{%- else -%}
24+
{%- set escape_chars = (var('escape_char_left', default_escape_char_left), var('escape_char_right', default_escape_char_right)) -%}
25+
{%- endif -%}
26+
27+
{%- if execute and warning_message -%}
28+
{%- do exceptions.warn(warning_message) -%}
29+
{%- endif -%}
30+
31+
{%- do return(escape_chars) -%}
1132

1233
{%- endmacro %}
1334

‎macros/materialisations/drop_temporary.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
DROP VIEW {{ tmp_relation }};
1414
{% elif target.type == 'sqlserver' %}
1515
DROP TABLE {{ tmp_relation }};
16+
{% elif target.type == 'postgres' %}
17+
DROP TABLE {{ tmp_relation }};
1618
{% endif %}
1719
{%- endcall %}
1820

‎macros/materialisations/incremental_pit_bridge_replace.sql

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,14 @@
44
*/
55

66
{% macro incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %}
7+
8+
{{ adapter.dispatch('incremental_pit_replace', 'automate_dv')(tmp_relation=tmp_relation,
9+
target_relation=target_relation,
10+
statement_name="main") }}
11+
12+
{%- endmacro -%}
13+
14+
{% macro default__incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %}
715
{%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
816
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
917

@@ -17,8 +25,28 @@
1725
{%- endmacro %}
1826

1927

28+
{% macro databricks__incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %}
29+
{%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
30+
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
31+
32+
INSERT OVERWRITE {{ target_relation }} ({{ dest_cols_csv }})
33+
SELECT {{ dest_cols_csv }}
34+
FROM {{ tmp_relation }};
35+
36+
{%- endmacro %}
37+
38+
2039

2140
{% macro incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %}
41+
42+
{{ adapter.dispatch('incremental_bridge_replace', 'automate_dv')(tmp_relation=tmp_relation,
43+
target_relation=target_relation,
44+
statement_name="main") }}
45+
46+
{%- endmacro -%}
47+
48+
49+
{% macro default__incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %}
2250
{%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
2351
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
2452

@@ -32,4 +60,15 @@
3260
{%- endmacro %}
3361

3462

63+
{% macro databricks__incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %}
64+
{%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
65+
{%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
66+
67+
INSERT OVERWRITE {{ target_relation }} ({{ dest_cols_csv }})
68+
SELECT {{ dest_cols_csv }}
69+
FROM {{ tmp_relation }}
70+
;
71+
{%- endmacro %}
72+
73+
3574

‎macros/materialisations/period_mat_helpers/check_datediff.sql

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,13 @@
55

66
{%- macro check_num_periods(start_date, stop_date, period) -%}
77

8-
{% set num_periods = adapter.dispatch('check_num_periods', 'automate_dv')(
9-
start_date=start_date,
10-
stop_date=stop_date,
11-
period=period) %}
8+
{% set num_periods = adapter.dispatch('check_num_periods',
9+
'automate_dv')(start_date=start_date,
10+
stop_date=stop_date,
11+
period=period) %}
1212

1313
{%- if num_periods > 100000 -%}
14-
{%- set error_message -%}
15-
'Max iterations is 100,000. Consider using a different datepart value (e.g. day)
16-
or loading data for a shorter time period.
17-
vault_insert_by materialisations are not intended for this purpose,
18-
please see https://automate_dv.readthedocs.io/en/latest/materialisations/'
19-
{%- endset -%}
20-
21-
{{- exceptions.raise_compiler_error(error_message) -}}
14+
{{ automate_dv.sqlserver_max_iterations_error() }}
2215
{%- endif -%}
2316

2417
{% do return(num_periods) %}
@@ -31,7 +24,7 @@
3124
SELECT {{ datediff('start_timestamp', 'stop_timestamp', period) }} AS NUM_PERIODS
3225
FROM
3326
(SELECT CAST('{{ start_date }}' AS {{ dbt.type_timestamp() }}) AS start_timestamp,
34-
CAST(NULLIF('{{ stop_date | lower }}', 'none') AS {{ dbt.type_timestamp() }}) AS stop_timestamp)
27+
CAST(NULLIF('{{ stop_date | lower }}', 'none') AS {{ dbt.type_timestamp() }}) AS stop_timestamp) AS SUBQUERY_ALIAS
3528
{% endset %}
3629
{% set num_periods_dict = automate_dv.get_query_results_as_dict(num_periods_check_sql) %}
3730
{% set num_periods = num_periods_dict['NUM_PERIODS'][0] | int %}

‎macros/materialisations/period_mat_helpers/error_messages.sql

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright (c) Business Thinking Ltd. 2019-2023
3+
* This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault
4+
*/
5+
6+
{%- macro sqlserver_datepart_too_small_error(period) -%}
7+
8+
{%- set error_message -%}
9+
This datepart ({{ period }}) is too small and cannot be used for this purpose in MS SQL Server, consider using a different datepart value (e.g. day).
10+
vault_insert_by materialisations are not intended for this purpose, please see https://automate-dv.readthedocs.io/en/latest/materialisations/
11+
{%- endset -%}
12+
13+
{{- exceptions.raise_compiler_error(error_message) -}}
14+
15+
{%- endmacro -%}
16+
17+
{%- macro sqlserver_max_iterations_error() -%}
18+
19+
{%- set error_message -%}
20+
Max iterations is 100,000. Consider using a different datepart value (e.g. day) or loading data for a shorter time period.
21+
vault_insert_by materialisations are not intended for this purpose, please see https://automate-dv.readthedocs.io/en/latest/materialisations/
22+
{%- endset -%}
23+
24+
{{- exceptions.raise_compiler_error(error_message) -}}
25+
26+
{%- endmacro -%}

0 commit comments

Comments
 (0)