Skip to content
This repository was archived by the owner on Oct 30, 2023. It is now read-only.

Commit fa36cfd

Browse files
yu-iskwyu-iskw
authored
Update models to dbt 1.3 (#53)
1 parent edb104b commit fa36cfd

File tree

19 files changed

+428
-7
lines changed

19 files changed

+428
-7
lines changed

‎dbt_artifacts_models/Makefile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,16 @@ setup:
44
dbt deps
55

66
build:
7-
dbt build --profiles-dir ./profiles --selector dbt-1.2
7+
dbt build --profiles-dir ./profiles --selector dbt-1.3
88

99
run:
10-
dbt run --profiles-dir ./profiles --selector dbt-1.2
10+
dbt run --profiles-dir ./profiles --selector dbt-1.3
1111

1212
compile:
1313
dbt compile --profiles-dir ./profiles
1414

1515
test:
16-
dbt test --profiles-dir profiles/ --selector dbt-1.2
16+
dbt test --profiles-dir profiles/ --selector dbt-1.4
1717

1818
docs-generate:
1919
dbt docs generate --profiles-dir profiles/

‎dbt_artifacts_models/dbt_project.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: dbt_artifacts_models
22

33
profile: dbt-artifacts-loader
44

5-
version: '1.2'
5+
version: '1.3'
66
config-version: 2
77
require-dbt-version: ">=1.0.0"
88

‎dbt_artifacts_models/models/marts/v4/run_results_v4/generic_test_run_results_v4/generic_test_run_results_v4.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ WITH run_results AS (
3636
LEFT OUTER JOIN {{ ref("parsed_generic_test_node_v6") }} AS generic_tests
3737
ON run_results.unique_id = generic_tests.unique_id
3838
AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, generic_tests.metadata.generated_at, DAY)) <= 2
39+
{% elif dbt_minor_version == "1.3" %}
40+
LEFT OUTER JOIN {{ ref("parsed_generic_test_node_v7") }} AS generic_tests
41+
ON run_results.unique_id = generic_tests.unique_id
42+
AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, generic_tests.metadata.generated_at, DAY)) <= 2
3943
{% else %}
4044
{{ exceptions.raise_compiler_error("Unexpected dbt version: " ~ dbt_minor_version) }}
4145
{% endif %}

‎dbt_artifacts_models/models/marts/v4/run_results_v4/model_run_results_v4/model_run_results_v4.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ WITH run_results AS (
3737
LEFT OUTER JOIN {{ ref("parsed_model_node_v6") }} AS models
3838
ON run_results.unique_id = models.unique_id
3939
AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, models.metadata.generated_at, DAY)) <= 2
40+
{% elif dbt_minor_version == "1.3" %}
41+
LEFT OUTER JOIN {{ ref("parsed_model_node_v7") }} AS models
42+
ON run_results.unique_id = models.unique_id
43+
AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, models.metadata.generated_at, DAY)) <= 2
4044
{% else %}
4145
{{ exceptions.raise_compiler_error("Unexpected dbt version: " ~ dbt_minor_version) }}
4246
{% endif %}

‎dbt_artifacts_models/models/marts/v4/run_results_v4/singular_test_run_results_v4/singular_test_run_results_v4.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ WITH run_results AS (
3737
LEFT OUTER JOIN {{ ref("parsed_singular_test_node_v6") }} AS singular_tests
3838
ON run_results.unique_id = singular_tests.unique_id
3939
AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, singular_tests.metadata.generated_at, DAY)) <= 2
40+
{% elif dbt_minor_version == "1.3" %}
41+
LEFT OUTER JOIN {{ ref("parsed_singular_test_node_v7") }} AS singular_tests
42+
ON run_results.unique_id = singular_tests.unique_id
43+
AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, singular_tests.metadata.generated_at, DAY)) <= 2
4044
{% else %}
4145
{{ exceptions.raise_compiler_error("Unexpected dbt version: " ~ dbt_minor_version) }}
4246
{% endif %}

‎dbt_artifacts_models/models/marts/v4/run_results_v4/snapshot_run_results_v4/snapshot_run_results_v4.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ WITH run_results AS (
3636
LEFT OUTER JOIN {{ ref("parsed_snapshot_node_v6") }} AS snapshots
3737
ON run_results.unique_id = snapshots.unique_id
3838
AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, snapshots.metadata.generated_at, DAY)) <= 2
39+
{% elif dbt_minor_version == "1.3" %}
40+
LEFT OUTER JOIN {{ ref("parsed_snapshot_node_v7") }} AS snapshots
41+
ON run_results.unique_id = snapshots.unique_id
42+
AND ABS(DATETIME_DIFF(run_results.metadata.generated_at, snapshots.metadata.generated_at, DAY)) <= 2
3943
{% else %}
4044
{{ exceptions.raise_compiler_error("Unexpected dbt version: " ~ dbt_minor_version) }}
4145
{% endif %}

‎dbt_artifacts_models/models/sources/v7/manifest_v7.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
version: 2
3+
4+
sources:
5+
- name: "{{ var('dbt_artifacts_loader')['dataset'] }}"
6+
database: "{{ var('dbt_artifacts_loader')['project'] }}"
7+
tables:
8+
- name: manifest_v7
9+
identifier: manifest_v7
10+
11+
columns:
12+
- name: loaded_at
13+
tests:
14+
- not_null

‎dbt_artifacts_models/models/staging/v7/manifest_v7/parsed_generic_test_node_v7/parsed_generic_test_node_v7.sql

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
{% set project = var('dbt_artifacts_loader')['project'] %}
2+
{% set dataset = var('dbt_artifacts_loader')['dataset'] %}
3+
4+
{{
5+
config(
6+
enabled=true,
7+
full_refresh=none,
8+
materialized="view",
9+
database=project,
10+
schema=dataset,
11+
alias="parsed_generic_test_node_v7",
12+
persist_docs={"relation": true, "columns": true},
13+
labels={
14+
"modeled_by": "dbt",
15+
"app": "dbt-artifacts-loader",
16+
},
17+
)
18+
}}
19+
20+
WITH generic_tests AS (
21+
SELECT
22+
t.* EXCEPT(depends_on),
23+
depends_on_node,
24+
depends_on_macro,
25+
FROM (
26+
SELECT
27+
loaded_at AS loaded_at,
28+
metadata AS metadata,
29+
node.key AS key,
30+
node.value.ParsedGenericTestNode.*,
31+
FROM {{ source(var('dbt_artifacts_loader')['dataset'], 'manifest_v7') }}
32+
, UNNEST(nodes) AS node
33+
) AS t
34+
CROSS JOIN UNNEST(depends_on.nodes.value) AS depends_on_node
35+
CROSS JOIN UNNEST(depends_on.macros.value) AS depends_on_macro
36+
)
37+
, generic_tests_with_models AS (
38+
SELECT
39+
generic_tests.*,
40+
(SELECT AS STRUCT models.*) AS depends_on_model,
41+
FROM generic_tests AS generic_tests
42+
LEFT OUTER JOIN {{ ref("parsed_model_node_v7") }} AS models
43+
ON generic_tests.metadata.invocation_id = models.metadata.invocation_id
44+
AND generic_tests.depends_on_node = models.unique_id
45+
)
46+
, remove_duplicates AS (
47+
SELECT
48+
ROW_NUMBER() OVER (PARTITION BY metadata.invocation_id, unique_id ORDER BY metadata.generated_at DESC) AS rank,
49+
*
50+
FROM generic_tests_with_models
51+
WHERE unique_id IS NOT NULL
52+
)
53+
54+
SELECT
55+
* EXCEPT(rank)
56+
FROM remove_duplicates
57+
WHERE rank = 1

‎dbt_artifacts_models/models/staging/v7/manifest_v7/parsed_generic_test_node_v7/schema.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
---
2+
version: 2
3+
4+
models:
5+
- name: parsed_generic_test_node_v7
6+
description: |
7+
This is a table to expand `manifest_v7.node.value.ParsedGenericTestNode`.
8+
The table exclude duplicates.
9+
10+
tests:
11+
- dbt_utils.unique_combination_of_columns:
12+
combination_of_columns:
13+
- metadata.invocation_id
14+
- unique_id
15+
16+
columns:
17+
- name: metadata.invocation_id
18+
description: "invocation ID"
19+
tests:
20+
- not_null
21+
- name: unique_id
22+
description: "unique ID"
23+
tests:
24+
- not_null

‎dbt_artifacts_models/models/staging/v7/manifest_v7/parsed_model_node_v7/parsed_model_node_v7.sql

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{% set project = var('dbt_artifacts_loader')['project'] %}
2+
{% set dataset = var('dbt_artifacts_loader')['dataset'] %}
3+
4+
{{
5+
config(
6+
enabled=true,
7+
full_refresh=none,
8+
materialized="view",
9+
database=project,
10+
schema=dataset,
11+
alias="parsed_model_node_v7",
12+
persist_docs={"relation": true, "columns": true},
13+
labels={
14+
"modeled_by": "dbt",
15+
"app": "dbt-artifacts-loader",
16+
},
17+
)
18+
}}
19+
20+
WITH expanded_artifacts AS (
21+
SELECT
22+
loaded_at AS loaded_at,
23+
metadata AS metadata,
24+
node.key AS key,
25+
node.value.ParsedModelNode.*,
26+
FROM {{ source(var('dbt_artifacts_loader')['dataset'], 'manifest_v7') }}
27+
, UNNEST(nodes) AS node
28+
)
29+
, remove_duplicates AS (
30+
SELECT
31+
ROW_NUMBER() OVER (PARTITION BY metadata.invocation_id, unique_id ORDER BY metadata.generated_at DESC) AS rank,
32+
*
33+
FROM expanded_artifacts
34+
WHERE unique_id IS NOT NULL
35+
)
36+
37+
SELECT
38+
* EXCEPT(rank)
39+
FROM remove_duplicates
40+
WHERE rank = 1

0 commit comments

Comments
 (0)