Skip to content

Commit 088ed54

Browse files
hlcianfagnaamotl
hlcianfagna
authored and
amotl
committed
dbt: Add macros overrides for CrateDB, and requirements.txt
This is a working constellation based on dbt-postgres 1.6.x. See also: https://community.cratedb.com/t/using-dbt-with-cratedb/1566
1 parent 39ebceb commit 088ed54

File tree

4 files changed

+195
-0
lines changed

4 files changed

+195
-0
lines changed

‎framework/dbt/basic/macros/catalog.sql

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{% macro postgres__get_catalog(information_schema, schemas) -%}
2+
3+
{%- call statement('catalog', fetch_result=True) -%}
4+
{#
5+
Derived from https://github.com/dbt-labs/dbt-core/blob/main/plugins/postgres/dbt/include/postgres/macros/catalog.sql
6+
#}
7+
{% set database = information_schema.database %}
8+
{{ adapter.verify_database(database) }}
9+
select
10+
'{{ database }}' as table_database,
11+
sch.nspname as table_schema,
12+
tbl.relname as table_name,
13+
case tbl.relkind
14+
when 'v' then 'VIEW'
15+
else 'BASE TABLE'
16+
end as table_type,
17+
tbl_desc.description as table_comment,
18+
col.attname as column_name,
19+
col.attnum as column_index,
20+
pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,
21+
col_desc.description as column_comment,
22+
pg_get_userbyid(tbl.relowner) as table_owner
23+
from pg_catalog.pg_namespace sch
24+
join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid
25+
join pg_catalog.pg_attribute col on col.attrelid = tbl.oid
26+
left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)
27+
left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)
28+
where (
29+
{%- for schema in schemas -%}
30+
upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
31+
{%- endfor -%}
32+
)
33+
and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables
34+
and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view
35+
and col.attnum > 0 -- negative numbers are used for system columns such as oid
36+
and not col.attisdropped -- column as not been dropped
37+
order by
38+
sch.nspname,
39+
tbl.relname,
40+
col.attnum
41+
{%- endcall -%}
42+
{{ return(load_result('catalog').table) }}
43+
{%- endmacro %}

‎framework/dbt/basic/macros/overrides.sql

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}
2+
{% set sql = "" %}
3+
{% if full_refresh %}
4+
{{ adapter.drop_relation(old_relation) }}
5+
{% set sql = create_csv_table(model, agate_table) %}
6+
{% else %}
7+
{{ adapter.truncate_relation(old_relation) }}
8+
{% set sql = "delete from " ~ old_relation %}
9+
{% endif %}
10+
11+
{{ return(sql) }}
12+
{% endmacro %}
13+
14+
{% macro generate_schema_name(custom_schema_name, node) -%}
15+
16+
{%- set default_schema = target.schema -%}
17+
{%- if custom_schema_name is none -%}
18+
19+
{{ default_schema }}
20+
21+
{%- else -%}
22+
23+
{{ custom_schema_name | trim }}
24+
25+
{%- endif -%}
26+
27+
{%- endmacro %}
28+
29+
{% macro postgres__create_schema(relation) -%}
30+
{%- call statement('create_schema') -%}
31+
/* schemas are not created in CrateDB */
32+
DROP TABLE IF EXISTS thisschemadefinitelydoesnotexits.thiswouldnotexist
33+
/* but we need to run something to not have just EOF */
34+
{% endcall %}
35+
{% endmacro %}
36+
37+
{% macro postgres__create_table_as(temporary, relation, sql) -%}
38+
{%- set unlogged = config.get('unlogged', default=false) -%}
39+
{%- set sql_header = config.get('sql_header', none) -%}
40+
41+
{{ sql_header if sql_header is not none }}
42+
43+
create table {{ relation }}
44+
as (
45+
{{ sql|replace('"crate".', "") }}
46+
);
47+
{%- endmacro %}
48+
49+
{% macro postgres__drop_schema(relation) -%}
50+
{% if relation.database -%}
51+
{{ adapter.verify_database(relation.database) }}
52+
{%- endif -%}
53+
{%- call statement('drop_schema') -%}
54+
/* schemas are not dropped in CrateDB */
55+
{%- endcall -%}
56+
{% endmacro %}
57+
58+
{% macro default__drop_relation(relation) -%}
59+
{% call statement('drop_relation', auto_begin=False) -%}
60+
drop {{ relation.type }} if exists "{{ relation.schema }}"."{{ relation.identifier }}"
61+
{%- endcall %}
62+
{% endmacro %}
63+
64+
{% macro default__drop_schema(relation) -%}
65+
{%- call statement('drop_schema') -%}
66+
/* schemas are not dropped in CrateDB */
67+
{% endcall %}
68+
{% endmacro %}
69+
70+
{% macro default__create_view_as(relation, sql) -%}
71+
{%- set sql_header = config.get('sql_header', none) -%}
72+
73+
{{ sql_header if sql_header is not none }}
74+
create view "{{ relation.schema }}"."{{ relation.identifier }}" as
75+
{{ sql|replace('"crate".', "") }}
76+
;
77+
{%- endmacro %}
78+
79+
{% macro postgres__rename_relation(from_relation, to_relation) -%}
80+
{% do drop_relation(to_relation) %}
81+
{% set schema_query = "SELECT table_type FROM information_schema.tables WHERE table_schema = '{}' AND table_name = '{}'".format(from_relation.schema, from_relation.identifier) %}
82+
{% set results = run_query(schema_query) %}
83+
{% if execute %}
84+
{% set results_list = results.columns[0].values() %}
85+
{% else %}
86+
{% set results_list = [] %}
87+
{% endif %}
88+
{% for relation_type in results_list %}
89+
{% if relation_type == 'VIEW' %}
90+
{% set view_query = "SELECT view_definition FROM information_schema.views WHERE table_schema = '{}' AND table_name = '{}'".format(from_relation.schema, from_relation.identifier) %}
91+
{% set view_definitions = run_query(view_query) %}
92+
{% if execute %}
93+
{% set view_definitions_list = view_definitions.columns[0].values() %}
94+
{% else %}
95+
{% set view_definitions_list = [] %}
96+
{% endif %}
97+
{% for view_definition in view_definitions_list %}
98+
{% call statement('drop_view') -%}
99+
DROP VIEW IF EXISTS {{ to_relation.schema }}.{{ to_relation.identifier }};
100+
{%- endcall %}
101+
{% call statement('create_view') -%}
102+
CREATE VIEW {{ to_relation.schema }}.{{ to_relation.identifier }} AS {{ view_definition }}
103+
{%- endcall %}
104+
{% call statement('drop_view') -%}
105+
DROP VIEW IF EXISTS {{ from_relation.schema }}.{{ from_relation.identifier }};
106+
{%- endcall %}
107+
{% endfor %}
108+
{% else %}
109+
{% call statement('rename_table') -%}
110+
ALTER TABLE {{ from_relation.schema }}.{{ from_relation.identifier }}
111+
RENAME TO {{ to_relation.identifier }}
112+
{%- endcall %}
113+
{% endif %}
114+
{% endfor %}
115+
{% endmacro %}
116+

‎framework/dbt/basic/macros/relations.sql

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{% macro postgres_get_relations () -%}
2+
{%- call statement('relations', fetch_result=True) -%}
3+
select 'mock' as referenced_schema,'referenced ' as referenced_name,
4+
'mock' as dependent_schema,'dependent' as dependent_name;
5+
{%- endcall -%}
6+
{{ return(load_result('relations').table) }}
7+
{% endmacro %}
8+
9+
{% macro postgres__list_relations_without_caching(schema_relation) %}
10+
{% call statement('list_relations_without_caching', fetch_result=True) -%}
11+
select
12+
'{{ schema_relation.database }}' as database,
13+
tablename as name,
14+
schemaname as schema,
15+
'table' as type
16+
from pg_tables
17+
where schemaname ilike '{{ schema_relation.schema }}'
18+
union all
19+
select
20+
'{{ schema_relation.database }}' as database,
21+
viewname as name,
22+
schemaname as schema,
23+
'view' as type
24+
from pg_views
25+
where schemaname ilike '{{ schema_relation.schema }}'
26+
{% endcall %}
27+
{{ return(load_result('list_relations_without_caching').table) }}
28+
{% endmacro %}
29+
30+
{% macro default__truncate_relation(relation) -%}
31+
{% call statement('truncate_relation') -%}
32+
delete from {{ relation }}
33+
{%- endcall %}
34+
{% endmacro %}

‎framework/dbt/basic/requirements.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
dbt-postgres<1.7
2+
snowplow-tracker<0.13

0 commit comments

Comments
 (0)