From 650aea56af7ece10bd066a486cc7648300d88305 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Wed, 8 May 2024 14:07:11 +0100 Subject: [PATCH 1/6] WIP --- .github/workflows/validate_schema.yml | 2 +- .../Schemas/dbtUnified_0.2.0.json | 55 +++++++++++--- .../Schemas/dbtUnified_0.3.0.json | 74 ++++++++++++++----- .../Schemas/dbtUnified_0.3.1.json | 74 ++++++++++++++----- .../Schemas/dbtUnified_0.4.0.json | 74 ++++++++++++++----- ...-schema => dbt-variables-meta-schema.json} | 34 +++++++++ 6 files changed, 246 insertions(+), 67 deletions(-) rename src/components/JsonSchemaValidator/{dbt-variables-meta-schema => dbt-variables-meta-schema.json} (74%) diff --git a/.github/workflows/validate_schema.yml b/.github/workflows/validate_schema.yml index add380ea7b..3e106dd68a 100644 --- a/.github/workflows/validate_schema.yml +++ b/.github/workflows/validate_schema.yml @@ -21,7 +21,7 @@ jobs: uses: GrantBirki/json-yaml-validate@v2.6.0 with: comment: "true" # enable comment mode - json_schema: ./src/components/JsonSchemaValidator/dbt-variables-meta-schema # validate JSON files against the schema + json_schema: ./src/components/JsonSchemaValidator/dbt-variables-meta-schema.json # validate JSON files against the schema base_dir: ./src/components/JsonSchemaValidator/Schemas/ json_schema_version: "draft-07" ajv_strict_mode: false diff --git a/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.2.0.json b/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.2.0.json index 3784e1a73f..c3c6e7f60d 100644 --- a/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.2.0.json +++ b/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.2.0.json @@ -16,10 +16,15 @@ "title": "SQL & Alias", "properties": { "sql": { - "type": "string" + "type": "string", + "title": "Field SQL", + "format": "sql", + "order": 1 }, "alias": { - "type": "string" + "type": "string", + "title": "Field Alias", + "order": 2 } }, "required": [ @@ -159,26 +164,33 @@ "name": { "type": "string", "title": "Name", - "description": "Name of your conversion type" + "description": "Name of your conversion type", + "order": 1 }, "condition": { "type": "string", "title": "Condition", - "description": "SQL condition e.g. event_name = 'page_view'" + "description": "SQL condition e.g. event_name = 'page_view'", + "format": "sql", + "order": 2 }, "value": { "type": "string", "title": "Value", - "description": "SQL value e.g. tr_total_base" + "description": "SQL value e.g. tr_total_base", + "format": "sql", + "order": 3 }, "default_value": { "type": "number", "title": "Default value", - "description": "Default value e.g. 0" + "description": "Default value e.g. 0", + "order": 4 }, "list_events": { "type": "boolean", - "title": "List all event ids?" + "title": "List all event ids?", + "order": 5 } } }, @@ -313,7 +325,8 @@ "packageDefault": "[ ] (no filter applied)", "group": "Contexts, Filters, and Logs", "items": { - "type": "string" + "type": "string", + "title": "App ID" } }, "snowplow__enable_consent": { @@ -467,10 +480,14 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", + "order": 2, + "title": "Field name", "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, @@ -487,6 +504,7 @@ "order": 47, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "SQL for your session identifier", "longDescription": "This allows you to override the `session_identifiers` SQL, to define completely custom SQL in order to build out a session identifier for your events. If you are interested in using this instead of providing identifiers through the `session_identifiers` variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details on how to do that.", "packageDefault": "", @@ -519,10 +537,14 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", + "title": "Field name", + "order": 2, "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, @@ -539,6 +561,7 @@ "order": 49, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "SQL for your user identifier", "longDescription": "This allows you to override the `user_identifiers` SQL, to define completely custom SQL in order to build out a user identifier for your events. If you are interested in using this instead of providing identifiers through the `user_identifiers` variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details on how to do that.", "packageDefault": "", @@ -549,6 +572,7 @@ "order": 50, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "Field used when stitching together users", "longDescription": "This is the user_id you want to stitch to sessions (and/or page views) with matching domain_userids. It supports raw `sql` expressions.", "packageDefault": "user_id", @@ -622,19 +646,26 @@ "properties": { "schema": { "type": "string", - "description": "Table name" + "title": "Table name", + "description": "Table name", + "order": 1 }, "prefix": { "type": "string", - "description": "Prefix to add to columns" + "title": "Column prefix", + "description": "Prefix to add to columns", + "order": 2 }, "alias": { "type": "string", - "description": "Table alias for the subquery" + "title": "CTE Alias", + "description": "Table alias for the subquery", + "order": 3 }, "single_entity": { "type": "boolean", - "title": "Is single entity?" + "title": "Is single entity?", + "order": 4 } }, "required": [ diff --git a/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.3.0.json b/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.3.0.json index abf1cba43d..c062f410d8 100644 --- a/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.3.0.json +++ b/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.3.0.json @@ -16,10 +16,15 @@ "title": "SQL & Alias", "properties": { "sql": { - "type": "string" + "type": "string", + "title": "Field SQL", + "format": "sql", + "order": 1 }, "alias": { - "type": "string" + "type": "string", + "title": "Field Alias", + "order": 2 } }, "required": [ @@ -47,13 +52,19 @@ "max", "count", "countd" - ] + ], + "title": "Aggregation type", + "order": 1 }, "field": { - "type": "string" + "type": "string", + "title": "Field to aggregate", + "order": 2 }, "alias": { - "type": "string" + "type": "string", + "title": "Aggregated field alias", + "order": 3 } }, "required": [ @@ -192,26 +203,33 @@ "name": { "type": "string", "title": "Name", - "description": "Name of your conversion type" + "description": "Name of your conversion type", + "order": 1 }, "condition": { "type": "string", "title": "Condition", - "description": "SQL condition e.g. event_name = 'page_view'" + "description": "SQL condition e.g. event_name = 'page_view'", + "format": "sql", + "order": 2 }, "value": { "type": "string", "title": "Value", - "description": "SQL value e.g. tr_total_base" + "description": "SQL value e.g. tr_total_base", + "format": "sql", + "order": 3 }, "default_value": { "type": "number", "title": "Default value", - "description": "Default value e.g. 0" + "description": "Default value e.g. 0", + "order": 4 }, "list_events": { "type": "boolean", - "title": "List all event ids?" + "title": "List all event ids?", + "order": 5 } } }, @@ -346,7 +364,8 @@ "packageDefault": "[ ] (no filter applied)", "group": "Contexts, Filters, and Logs", "items": { - "type": "string" + "type": "string", + "title": "App ID" } }, "snowplow__enable_consent": { @@ -500,10 +519,14 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", + "order": 2, + "title": "Field name", "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, @@ -520,6 +543,7 @@ "order": 47, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "SQL for your session identifier", "longDescription": "This allows you to override the `session_identifiers` SQL, to define completely custom SQL in order to build out a session identifier for your events. If you are interested in using this instead of providing identifiers through the `session_identifiers` variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details on how to do that.", "packageDefault": "", @@ -552,10 +576,14 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", + "title": "Field name", + "order": 2, "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, @@ -572,6 +600,7 @@ "order": 49, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "SQL for your user identifier", "longDescription": "This allows you to override the `user_identifiers` SQL, to define completely custom SQL in order to build out a user identifier for your events. If you are interested in using this instead of providing identifiers through the `user_identifiers` variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details on how to do that.", "packageDefault": "", @@ -582,6 +611,7 @@ "order": 50, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "Field used when stitching together users", "longDescription": "This is the user_id you want to stitch to sessions (and/or page views) with matching domain_userids. It supports raw `sql` expressions.", "packageDefault": "user_id", @@ -655,19 +685,26 @@ "properties": { "schema": { "type": "string", - "description": "Table name" + "title": "Table name", + "description": "Table name", + "order": 1 }, "prefix": { "type": "string", - "description": "Prefix to add to columns" + "title": "Column prefix", + "description": "Prefix to add to columns", + "order": 2 }, "alias": { "type": "string", - "description": "Table alias for the subquery" + "title": "CTE Alias", + "description": "Table alias for the subquery", + "order": 3 }, "single_entity": { "type": "boolean", - "title": "Is single entity?" + "title": "Is single entity?", + "order": 4 } }, "required": [ @@ -814,6 +851,7 @@ "consoleGroup": "advanced", "type": "string", "title": "Custom SQL", + "format": "sql", "group": "Operation and Logic", "longDescription": "This allows you to introduce custom sql to the `snowplow_unified_events_this_run` table, which you can then leverage in downstream models. For more information on the usage, see the following page on the [advanced usage of the utils package](/docs/modeling-your-data/modeling-your-data-with-dbt/dbt-custom-models/examples/additional-sql-on-events-this-run/).", "packageDefault": "", @@ -827,7 +865,7 @@ "group": "Contexts, Filters, and Logs", "longDescription": "Aggregations to calculate as part of the the derived table. A list of dictionaries defining the type (sum, avg, min, max, count, countd), field to aggregate, and what to alias the column.", "packageDefault": "[ ] (no aggregations)", - "$ref": "#/definitions/passthrough_vars" + "$ref": "#/definitions/aggregation_vars" }, "snowplow__session_aggregations": { "recommendFullRefresh": false, @@ -837,7 +875,7 @@ "group": "Contexts, Filters, and Logs", "longDescription": "Aggregations to calculate as part of the the derived table. A list of dictionaries defining the type (sum, avg, min, max, count, countd), field to aggregate, and what to alias the column.", "packageDefault": "[ ] (no aggregations)", - "$ref": "#/definitions/passthrough_vars" + "$ref": "#/definitions/aggregation_vars" }, "snowplow__user_aggregations": { "recommendFullRefresh": false, @@ -847,7 +885,7 @@ "group": "Contexts, Filters, and Logs", "longDescription": " Aggregations to calculate as part of the the derived table. A list of dictionaries defining the type (sum, avg, min, max, count, countd), field to aggregate, and what to alias the column.", "packageDefault": "[ ] (no aggregations)", - "$ref": "#/definitions/passthrough_vars" + "$ref": "#/definitions/aggregation_vars" }, "snowplow__grant_select_to": { "recommendFullRefresh": false, diff --git a/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.3.1.json b/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.3.1.json index abf1cba43d..c062f410d8 100644 --- a/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.3.1.json +++ b/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.3.1.json @@ -16,10 +16,15 @@ "title": "SQL & Alias", "properties": { "sql": { - "type": "string" + "type": "string", + "title": "Field SQL", + "format": "sql", + "order": 1 }, "alias": { - "type": "string" + "type": "string", + "title": "Field Alias", + "order": 2 } }, "required": [ @@ -47,13 +52,19 @@ "max", "count", "countd" - ] + ], + "title": "Aggregation type", + "order": 1 }, "field": { - "type": "string" + "type": "string", + "title": "Field to aggregate", + "order": 2 }, "alias": { - "type": "string" + "type": "string", + "title": "Aggregated field alias", + "order": 3 } }, "required": [ @@ -192,26 +203,33 @@ "name": { "type": "string", "title": "Name", - "description": "Name of your conversion type" + "description": "Name of your conversion type", + "order": 1 }, "condition": { "type": "string", "title": "Condition", - "description": "SQL condition e.g. event_name = 'page_view'" + "description": "SQL condition e.g. event_name = 'page_view'", + "format": "sql", + "order": 2 }, "value": { "type": "string", "title": "Value", - "description": "SQL value e.g. tr_total_base" + "description": "SQL value e.g. tr_total_base", + "format": "sql", + "order": 3 }, "default_value": { "type": "number", "title": "Default value", - "description": "Default value e.g. 0" + "description": "Default value e.g. 0", + "order": 4 }, "list_events": { "type": "boolean", - "title": "List all event ids?" + "title": "List all event ids?", + "order": 5 } } }, @@ -346,7 +364,8 @@ "packageDefault": "[ ] (no filter applied)", "group": "Contexts, Filters, and Logs", "items": { - "type": "string" + "type": "string", + "title": "App ID" } }, "snowplow__enable_consent": { @@ -500,10 +519,14 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", + "order": 2, + "title": "Field name", "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, @@ -520,6 +543,7 @@ "order": 47, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "SQL for your session identifier", "longDescription": "This allows you to override the `session_identifiers` SQL, to define completely custom SQL in order to build out a session identifier for your events. If you are interested in using this instead of providing identifiers through the `session_identifiers` variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details on how to do that.", "packageDefault": "", @@ -552,10 +576,14 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", + "title": "Field name", + "order": 2, "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, @@ -572,6 +600,7 @@ "order": 49, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "SQL for your user identifier", "longDescription": "This allows you to override the `user_identifiers` SQL, to define completely custom SQL in order to build out a user identifier for your events. If you are interested in using this instead of providing identifiers through the `user_identifiers` variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details on how to do that.", "packageDefault": "", @@ -582,6 +611,7 @@ "order": 50, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "Field used when stitching together users", "longDescription": "This is the user_id you want to stitch to sessions (and/or page views) with matching domain_userids. It supports raw `sql` expressions.", "packageDefault": "user_id", @@ -655,19 +685,26 @@ "properties": { "schema": { "type": "string", - "description": "Table name" + "title": "Table name", + "description": "Table name", + "order": 1 }, "prefix": { "type": "string", - "description": "Prefix to add to columns" + "title": "Column prefix", + "description": "Prefix to add to columns", + "order": 2 }, "alias": { "type": "string", - "description": "Table alias for the subquery" + "title": "CTE Alias", + "description": "Table alias for the subquery", + "order": 3 }, "single_entity": { "type": "boolean", - "title": "Is single entity?" + "title": "Is single entity?", + "order": 4 } }, "required": [ @@ -814,6 +851,7 @@ "consoleGroup": "advanced", "type": "string", "title": "Custom SQL", + "format": "sql", "group": "Operation and Logic", "longDescription": "This allows you to introduce custom sql to the `snowplow_unified_events_this_run` table, which you can then leverage in downstream models. For more information on the usage, see the following page on the [advanced usage of the utils package](/docs/modeling-your-data/modeling-your-data-with-dbt/dbt-custom-models/examples/additional-sql-on-events-this-run/).", "packageDefault": "", @@ -827,7 +865,7 @@ "group": "Contexts, Filters, and Logs", "longDescription": "Aggregations to calculate as part of the the derived table. A list of dictionaries defining the type (sum, avg, min, max, count, countd), field to aggregate, and what to alias the column.", "packageDefault": "[ ] (no aggregations)", - "$ref": "#/definitions/passthrough_vars" + "$ref": "#/definitions/aggregation_vars" }, "snowplow__session_aggregations": { "recommendFullRefresh": false, @@ -837,7 +875,7 @@ "group": "Contexts, Filters, and Logs", "longDescription": "Aggregations to calculate as part of the the derived table. A list of dictionaries defining the type (sum, avg, min, max, count, countd), field to aggregate, and what to alias the column.", "packageDefault": "[ ] (no aggregations)", - "$ref": "#/definitions/passthrough_vars" + "$ref": "#/definitions/aggregation_vars" }, "snowplow__user_aggregations": { "recommendFullRefresh": false, @@ -847,7 +885,7 @@ "group": "Contexts, Filters, and Logs", "longDescription": " Aggregations to calculate as part of the the derived table. A list of dictionaries defining the type (sum, avg, min, max, count, countd), field to aggregate, and what to alias the column.", "packageDefault": "[ ] (no aggregations)", - "$ref": "#/definitions/passthrough_vars" + "$ref": "#/definitions/aggregation_vars" }, "snowplow__grant_select_to": { "recommendFullRefresh": false, diff --git a/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.4.0.json b/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.4.0.json index abf1cba43d..c062f410d8 100644 --- a/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.4.0.json +++ b/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.4.0.json @@ -16,10 +16,15 @@ "title": "SQL & Alias", "properties": { "sql": { - "type": "string" + "type": "string", + "title": "Field SQL", + "format": "sql", + "order": 1 }, "alias": { - "type": "string" + "type": "string", + "title": "Field Alias", + "order": 2 } }, "required": [ @@ -47,13 +52,19 @@ "max", "count", "countd" - ] + ], + "title": "Aggregation type", + "order": 1 }, "field": { - "type": "string" + "type": "string", + "title": "Field to aggregate", + "order": 2 }, "alias": { - "type": "string" + "type": "string", + "title": "Aggregated field alias", + "order": 3 } }, "required": [ @@ -192,26 +203,33 @@ "name": { "type": "string", "title": "Name", - "description": "Name of your conversion type" + "description": "Name of your conversion type", + "order": 1 }, "condition": { "type": "string", "title": "Condition", - "description": "SQL condition e.g. event_name = 'page_view'" + "description": "SQL condition e.g. event_name = 'page_view'", + "format": "sql", + "order": 2 }, "value": { "type": "string", "title": "Value", - "description": "SQL value e.g. tr_total_base" + "description": "SQL value e.g. tr_total_base", + "format": "sql", + "order": 3 }, "default_value": { "type": "number", "title": "Default value", - "description": "Default value e.g. 0" + "description": "Default value e.g. 0", + "order": 4 }, "list_events": { "type": "boolean", - "title": "List all event ids?" + "title": "List all event ids?", + "order": 5 } } }, @@ -346,7 +364,8 @@ "packageDefault": "[ ] (no filter applied)", "group": "Contexts, Filters, and Logs", "items": { - "type": "string" + "type": "string", + "title": "App ID" } }, "snowplow__enable_consent": { @@ -500,10 +519,14 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", + "order": 2, + "title": "Field name", "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, @@ -520,6 +543,7 @@ "order": 47, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "SQL for your session identifier", "longDescription": "This allows you to override the `session_identifiers` SQL, to define completely custom SQL in order to build out a session identifier for your events. If you are interested in using this instead of providing identifiers through the `session_identifiers` variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details on how to do that.", "packageDefault": "", @@ -552,10 +576,14 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", + "title": "Field name", + "order": 2, "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, @@ -572,6 +600,7 @@ "order": 49, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "SQL for your user identifier", "longDescription": "This allows you to override the `user_identifiers` SQL, to define completely custom SQL in order to build out a user identifier for your events. If you are interested in using this instead of providing identifiers through the `user_identifiers` variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details on how to do that.", "packageDefault": "", @@ -582,6 +611,7 @@ "order": 50, "consoleGroup": "advanced", "type": "string", + "format": "sql", "title": "Field used when stitching together users", "longDescription": "This is the user_id you want to stitch to sessions (and/or page views) with matching domain_userids. It supports raw `sql` expressions.", "packageDefault": "user_id", @@ -655,19 +685,26 @@ "properties": { "schema": { "type": "string", - "description": "Table name" + "title": "Table name", + "description": "Table name", + "order": 1 }, "prefix": { "type": "string", - "description": "Prefix to add to columns" + "title": "Column prefix", + "description": "Prefix to add to columns", + "order": 2 }, "alias": { "type": "string", - "description": "Table alias for the subquery" + "title": "CTE Alias", + "description": "Table alias for the subquery", + "order": 3 }, "single_entity": { "type": "boolean", - "title": "Is single entity?" + "title": "Is single entity?", + "order": 4 } }, "required": [ @@ -814,6 +851,7 @@ "consoleGroup": "advanced", "type": "string", "title": "Custom SQL", + "format": "sql", "group": "Operation and Logic", "longDescription": "This allows you to introduce custom sql to the `snowplow_unified_events_this_run` table, which you can then leverage in downstream models. For more information on the usage, see the following page on the [advanced usage of the utils package](/docs/modeling-your-data/modeling-your-data-with-dbt/dbt-custom-models/examples/additional-sql-on-events-this-run/).", "packageDefault": "", @@ -827,7 +865,7 @@ "group": "Contexts, Filters, and Logs", "longDescription": "Aggregations to calculate as part of the the derived table. A list of dictionaries defining the type (sum, avg, min, max, count, countd), field to aggregate, and what to alias the column.", "packageDefault": "[ ] (no aggregations)", - "$ref": "#/definitions/passthrough_vars" + "$ref": "#/definitions/aggregation_vars" }, "snowplow__session_aggregations": { "recommendFullRefresh": false, @@ -837,7 +875,7 @@ "group": "Contexts, Filters, and Logs", "longDescription": "Aggregations to calculate as part of the the derived table. A list of dictionaries defining the type (sum, avg, min, max, count, countd), field to aggregate, and what to alias the column.", "packageDefault": "[ ] (no aggregations)", - "$ref": "#/definitions/passthrough_vars" + "$ref": "#/definitions/aggregation_vars" }, "snowplow__user_aggregations": { "recommendFullRefresh": false, @@ -847,7 +885,7 @@ "group": "Contexts, Filters, and Logs", "longDescription": " Aggregations to calculate as part of the the derived table. A list of dictionaries defining the type (sum, avg, min, max, count, countd), field to aggregate, and what to alias the column.", "packageDefault": "[ ] (no aggregations)", - "$ref": "#/definitions/passthrough_vars" + "$ref": "#/definitions/aggregation_vars" }, "snowplow__grant_select_to": { "recommendFullRefresh": false, diff --git a/src/components/JsonSchemaValidator/dbt-variables-meta-schema b/src/components/JsonSchemaValidator/dbt-variables-meta-schema.json similarity index 74% rename from src/components/JsonSchemaValidator/dbt-variables-meta-schema rename to src/components/JsonSchemaValidator/dbt-variables-meta-schema.json index 3b9dd2337b..e25b15ef0c 100644 --- a/src/components/JsonSchemaValidator/dbt-variables-meta-schema +++ b/src/components/JsonSchemaValidator/dbt-variables-meta-schema.json @@ -15,6 +15,9 @@ "patternProperties": { "^snowplow__": { "type": "object", + "additionalProperties": { + "$ref": "#" + }, "properties": { "recommendFullRefresh": { "type": "boolean" @@ -65,6 +68,24 @@ "Databricks", "Bigquery" ] + }, + "minimum": { + "type": "number" + }, + "maximum": { + "type": "number" + }, + "minItems": { + "type": "number" + }, + "maxItems": { + "type": "number" + }, + "format": { + "type": "string" + }, + "$ref": { + "type": "string" } }, "oneOf": [ @@ -94,6 +115,19 @@ } ], "default": {} + }, + "^.*": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "order": { + "type": "number" + } + }, + "additionalProperties": true, + "required": ["title", "order"] } }, "default": {} From 8c259ee82b2f1ea1401dd86df868333e9528a91a Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Wed, 8 May 2024 14:16:56 +0100 Subject: [PATCH 2/6] add exclude file? --- .github/workflows/validate_schema.yml | 1 + .../Schemas/exclude_from_checks.txt | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt diff --git a/.github/workflows/validate_schema.yml b/.github/workflows/validate_schema.yml index 3e106dd68a..ae42d59cf4 100644 --- a/.github/workflows/validate_schema.yml +++ b/.github/workflows/validate_schema.yml @@ -25,3 +25,4 @@ jobs: base_dir: ./src/components/JsonSchemaValidator/Schemas/ json_schema_version: "draft-07" ajv_strict_mode: false + exclude_file: ./src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt diff --git a/src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt b/src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt new file mode 100644 index 0000000000..d872fed5b5 --- /dev/null +++ b/src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt @@ -0,0 +1,21 @@ +src/components/JsonSchemaValidator/Schemas/dbtAttribution_0.2.0.json +src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.5.4.json +src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.6.0.json +src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.6.1.json +src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.7.0.json +src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.7.1.json +src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.8.0.json +src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.8.1.json +src/components/JsonSchemaValidator/Schemas/dbtFractribution_0.3.5.json +src/components/JsonSchemaValidator/Schemas/dbtFractribution_0.3.6.json +src/components/JsonSchemaValidator/Schemas/dbtMediaPlayer_0.6.1.json +src/components/JsonSchemaValidator/Schemas/dbtMediaPlayer_0.7.0.json +src/components/JsonSchemaValidator/Schemas/dbtMediaPlayer_0.7.1.json +src/components/JsonSchemaValidator/Schemas/dbtMobile* +src/components/JsonSchemaValidator/Schemas/dbtNormalize_0.3.5.json +src/components/JsonSchemaValidator/Schemas/dbtUnified_0.1.2.json +src/components/JsonSchemaValidator/Schemas/dbtUtils_0.15.2.json +src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.0.json +src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.1.json +src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.2.json +src/components/JsonSchemaValidator/Schemas/dbtWeb* From 0ba41f9dfe4838705a94b8a36787f84a079e5111 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Wed, 8 May 2024 14:20:42 +0100 Subject: [PATCH 3/6] try this --- .github/workflows/validate_schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_schema.yml b/.github/workflows/validate_schema.yml index ae42d59cf4..842eefec6d 100644 --- a/.github/workflows/validate_schema.yml +++ b/.github/workflows/validate_schema.yml @@ -25,4 +25,4 @@ jobs: base_dir: ./src/components/JsonSchemaValidator/Schemas/ json_schema_version: "draft-07" ajv_strict_mode: false - exclude_file: ./src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt + exclude_file: src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt From afebbbf5e327d1ce96e317cd8284bce1f24e7b95 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Wed, 8 May 2024 14:28:19 +0100 Subject: [PATCH 4/6] fix only top versions --- .../Schemas/dbtEcommerce_0.8.2.json | 34 +++++++++++--- .../Schemas/dbtMediaPlayer_0.7.2.json | 28 ++++++++--- .../Schemas/dbtUnified_0.4.0.json | 2 +- .../Schemas/dbtUtils_0.16.4.json | 46 ++++++++++++++----- .../Schemas/exclude_from_checks.txt | 1 + 5 files changed, 86 insertions(+), 25 deletions(-) diff --git a/src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.8.2.json b/src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.8.2.json index 05c626e736..aae7b66898 100644 --- a/src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.8.2.json +++ b/src/components/JsonSchemaValidator/Schemas/dbtEcommerce_0.8.2.json @@ -16,10 +16,15 @@ "title": "SQL & Alias", "properties": { "sql": { - "type": "string" + "type": "string", + "title": "Field SQL", + "format": "sql", + "order": 1 }, "alias": { - "type": "string" + "type": "string", + "title": "Field Alias", + "order": 2 } }, "required": [ @@ -358,11 +363,15 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", - "description": "The name of the field to use as session identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" + "order": 2, + "title": "Field name", + "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, "required": [ @@ -410,10 +419,14 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", + "title": "Field name", + "order": 2, "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, @@ -514,19 +527,26 @@ "properties": { "schema": { "type": "string", - "description": "Table name" + "title": "Table name", + "description": "Table name", + "order": 1 }, "prefix": { "type": "string", - "description": "Prefix to add to columns" + "title": "Column prefix", + "description": "Prefix to add to columns", + "order": 2 }, "alias": { "type": "string", - "description": "Table alias for the subquery" + "title": "CTE Alias", + "description": "Table alias for the subquery", + "order": 3 }, "single_entity": { "type": "boolean", - "title": "Is single entity?" + "title": "Is single entity?", + "order": 4 } }, "required": [ diff --git a/src/components/JsonSchemaValidator/Schemas/dbtMediaPlayer_0.7.2.json b/src/components/JsonSchemaValidator/Schemas/dbtMediaPlayer_0.7.2.json index 20d0c20213..134c6ec05f 100644 --- a/src/components/JsonSchemaValidator/Schemas/dbtMediaPlayer_0.7.2.json +++ b/src/components/JsonSchemaValidator/Schemas/dbtMediaPlayer_0.7.2.json @@ -230,6 +230,7 @@ }, "surrogate_key_treat_nulls_as_empty_strings": { "type": "boolean", + "order": 999, "title": "Surrogate Key Treat Nulls as Empty Strings", "group": "Operation and Logic", "longDescription": "Passed through to `dbt_utils` to match legacy surrogate key behavior.", @@ -253,11 +254,15 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", - "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" + "order": 2, + "title": "Field name", + "description": "The name of the field to use as session identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, "required": [ @@ -305,10 +310,14 @@ "properties": { "schema": { "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { "type": "string", + "order": 2, + "title": "Field name", "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, @@ -512,22 +521,29 @@ "items": { "type": "object", "title": "Entity or SDE", - "properties": { + "properties": { "schema": { "type": "string", - "description": "Table name" + "title": "Table name", + "description": "Table name", + "order": 1 }, "prefix": { "type": "string", - "description": "Prefix to add to columns" + "title": "Column prefix", + "description": "Prefix to add to columns", + "order": 2 }, "alias": { "type": "string", - "description": "Table alias for the subquery" + "title": "CTE Alias", + "description": "Table alias for the subquery", + "order": 3 }, "single_entity": { "type": "boolean", - "title": "Is single entity?" + "title": "Is single entity?", + "order": 4 } }, "required": [ diff --git a/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.4.0.json b/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.4.0.json index c062f410d8..017b7083e3 100644 --- a/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.4.0.json +++ b/src/components/JsonSchemaValidator/Schemas/dbtUnified_0.4.0.json @@ -527,7 +527,7 @@ "type": "string", "order": 2, "title": "Field name", - "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" + "description": "The name of the field to use as session identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, "required": [ diff --git a/src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.4.json b/src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.4.json index a71e16ff17..92df9b620d 100644 --- a/src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.4.json +++ b/src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.4.json @@ -16,10 +16,15 @@ "title": "SQL & Alias", "properties": { "sql": { - "type": "string" + "type": "string", + "title": "Field SQL", + "format": "sql", + "order": 1 }, "alias": { - "type": "string" + "type": "string", + "title": "Field Alias", + "order": 2 } }, "required": [ @@ -203,10 +208,16 @@ "title": "Identifier", "properties": { "schema": { - "type": "string" + "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, + "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { - "type": "string" + "type": "string", + "order": 2, + "title": "Field name", + "description": "The name of the field to use as session identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, "required": [ @@ -253,10 +264,16 @@ "title": "Identifier", "properties": { "schema": { - "type": "string" + "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, + "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" }, "field": { - "type": "string" + "type": "string", + "order": 2, + "title": "Field name", + "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" } }, "required": [ @@ -295,19 +312,26 @@ "properties": { "schema": { "type": "string", - "description": "Table name" + "title": "Table name", + "description": "Table name", + "order": 1 }, "prefix": { "type": "string", - "description": "Prefix to add to columns" + "title": "Column prefix", + "description": "Prefix to add to columns", + "order": 2 }, "alias": { "type": "string", - "description": "Table alias for the subquery" + "title": "CTE Alias", + "description": "Table alias for the subquery", + "order": 3 }, "single_entity": { "type": "boolean", - "title": "Is single entity?" + "title": "Is single entity?", + "order": 4 } }, "required": [ @@ -319,4 +343,4 @@ "uniqueItems": true } } -} \ No newline at end of file +} diff --git a/src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt b/src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt index d872fed5b5..ff35c4d62a 100644 --- a/src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt +++ b/src/components/JsonSchemaValidator/Schemas/exclude_from_checks.txt @@ -18,4 +18,5 @@ src/components/JsonSchemaValidator/Schemas/dbtUtils_0.15.2.json src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.0.json src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.1.json src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.2.json +src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.3.json src/components/JsonSchemaValidator/Schemas/dbtWeb* From d43b7df82dd45c8883c17ced2cb42e9f208cead0 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Wed, 8 May 2024 14:33:39 +0100 Subject: [PATCH 5/6] make date required for attribution --- .../JsonSchemaValidator/Schemas/dbtAttribution_0.2.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/JsonSchemaValidator/Schemas/dbtAttribution_0.2.0.json b/src/components/JsonSchemaValidator/Schemas/dbtAttribution_0.2.0.json index d3e3628b4a..7f8d407d9f 100644 --- a/src/components/JsonSchemaValidator/Schemas/dbtAttribution_0.2.0.json +++ b/src/components/JsonSchemaValidator/Schemas/dbtAttribution_0.2.0.json @@ -323,7 +323,7 @@ "snowplow__attribution_start_date": { "recommendFullRefresh": true, "order": 7, - "consoleGroup": "basic", + "consoleGroup": "required", "type": "string", "format": "date", "title": "Attribution Start Date", From 61a5d924e1eacd84cdbcd7dc0fea1e56b8dfe397 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Thu, 9 May 2024 10:18:46 +0100 Subject: [PATCH 6/6] Add utils release --- .../dbt-configuration/utils/index.mdx | 2 +- src/componentVersions.js | 2 +- .../Schemas/dbtUtils_0.16.5.json | 346 ++++++++++++++++++ 3 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.5.json diff --git a/docs/modeling-your-data/modeling-your-data-with-dbt/dbt-configuration/utils/index.mdx b/docs/modeling-your-data/modeling-your-data-with-dbt/dbt-configuration/utils/index.mdx index 3a97bac2c2..09a4ce756e 100644 --- a/docs/modeling-your-data/modeling-your-data-with-dbt/dbt-configuration/utils/index.mdx +++ b/docs/modeling-your-data/modeling-your-data-with-dbt/dbt-configuration/utils/index.mdx @@ -35,7 +35,7 @@ export const printYamlVariables = (data) => { export const MyMdxComponent = () => { const schemaName = 'dbtUtils'; - const versions = ['0.16.4', '0.16.3', '0.16.2', '0.16.1', '0.16.0', '0.15.2']; + const versions = ['0.16.5', '0.16.4', '0.16.3', '0.16.2', '0.16.1', '0.16.0', '0.15.2']; const label = 'dbt utils version'; const output = printYamlVariables; return ( diff --git a/src/componentVersions.js b/src/componentVersions.js index 4127778481..50de8ba6ea 100644 --- a/src/componentVersions.js +++ b/src/componentVersions.js @@ -45,7 +45,7 @@ export const versions = { dbtSnowplowUnified: '0.4.0', dbtSnowplowWeb: '1.0.0', dbtSnowplowMobile: '1.0.0', - dbtSnowplowUtils: '0.16.4', + dbtSnowplowUtils: '0.16.5', dbtSnowplowMediaPlayer: '0.7.2', dbtSnowplowNormalize: '0.3.5', dbtSnowplowFractribution: '0.3.6', diff --git a/src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.5.json b/src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.5.json new file mode 100644 index 0000000000..92df9b620d --- /dev/null +++ b/src/components/JsonSchemaValidator/Schemas/dbtUtils_0.16.5.json @@ -0,0 +1,346 @@ +{ + "definitions": { + "passthrough_vars": { + "type": "array", + "description": "> Click the plus sign to add a new entry", + "minItems": 0, + "items": { + "title": "Type", + "oneOf": [ + { + "type": "string", + "title": "Column Name" + }, + { + "type": "object", + "title": "SQL & Alias", + "properties": { + "sql": { + "type": "string", + "title": "Field SQL", + "format": "sql", + "order": 1 + }, + "alias": { + "type": "string", + "title": "Field Alias", + "order": 2 + } + }, + "required": [ + "sql", + "alias" + ], + "additionalProperties": false + } + ] + }, + "uniqueItems": true + } + }, + "type": "object", + "properties": { + "snowplow__database": { + "recommendFullRefresh": true, + "order": 1, + "consoleGroup": "required", + "type": "string", + "title": "Database", + "description": "Database that contains your atomic events", + "longDescription": "The database that contains your atomic events table.", + "packageDefault": "target.database", + "group": "Warehouse and Tracker" + }, + "snowplow__dev_target_name": { + "recommendFullRefresh": false, + "order": 87, + "consoleGroup": "advanced", + "type": "string", + "title": "Dev Target", + "description": "Target name of your development environment as defined in your `profiles.yml` file", + "longDescription": "The [target name](https://docs.getdbt.com/docs/core/connect-data-platform/profiles.yml) of your development environment as defined in your `profiles.yml` file. See the [Manifest Tables](/docs/modeling-your-data/modeling-your-data-with-dbt/package-mechanics/manifest-tables/) section for more details.", + "packageDefault": "dev", + "group": "Warehouse and Tracker" + }, + "snowplow__events_table": { + "recommendFullRefresh": true, + "order": 5, + "consoleGroup": "required", + "type": "string", + "title": "Events Table", + "description": "The name of the table that contains your atomic events", + "longDescription": "The name of the table that contains your atomic events.", + "packageDefault": "events", + "group": "Warehouse and Tracker" + }, + "snowplow__events_schema": { + "recommendFullRefresh": true, + "order": 4, + "consoleGroup": "basic", + "type": "string", + "title": "Events Schema", + "description": "The name of the schema that contains your atomic events", + "longDescription": "The name of the schema that contains your atomic events.", + "packageDefault": "events", + "group": "Warehouse and Tracker" + }, + "snowplow__allow_refresh": { + "recommendFullRefresh": true, + "order": 39, + "consoleGroup": "advanced", + "type": "boolean", + "title": "Allow Refresh", + "group": "Operation and Logic", + "longDescription": "Used as the default value to return from the `allow_refresh()` macro. This macro determines whether the manifest tables can be refreshed or not, depending on your environment. See the [Manifest Tables](/docs/modeling-your-data/modeling-your-data-with-dbt/package-mechanics/manifest-tables/) section for more details.", + "packageDefault": "false" + }, + "snowplow__backfill_limit_days": { + "recommendFullRefresh": false, + "order": 41, + "consoleGroup": "advanced", + "type": "number", + "minimum": 0, + "title": "Backfill Limit", + "group": "Operation and Logic", + "longDescription": "The maximum numbers of days of new data to be processed since the latest event processed. Please refer to the [incremental logic](docs/modeling-your-data/modeling-your-data-with-dbt/package-mechanics/incremental-processing/#package-state) section for more details.", + "packageDefault": "30", + "description": "The maximum numbers of days of new data to be processed since the latest event processed" + }, + "snowplow__custom_sql": { + "recommendFullRefresh": false, + "order": 84, + "consoleGroup": "advanced", + "type": "string", + "title": "Custom SQL", + "group": "Operation and Logic", + "longDescription": "This allows you to introduce custom sql to the `snowplow_base_events_this_run` table, which you can then leverage in downstream models. For more information on the usage, see the following page on the [advanced usage of the utils package](/docs/modeling-your-data/modeling-your-data-with-dbt/dbt-custom-models/examples/additional-sql-on-events-this-run/).", + "packageDefault": "", + "description": "Custom SQL for your base events this run table." + }, + "snowplow__days_late_allowed": { + "recommendFullRefresh": true, + "order": 42, + "consoleGroup": "advanced", + "type": "number", + "minimum": 0, + "title": "Days Late Allowed", + "group": "Operation and Logic", + "longDescription": "The maximum allowed number of days between the event creation and it being sent to the collector. Exists to reduce lengthy table scans that can occur as a result of late arriving data.", + "packageDefault": "3", + "description": "The maximum allowed number of days between the event creation and it being sent to the collector" + }, + "snowplow__max_session_days": { + "recommendFullRefresh": true, + "order": 110, + "consoleGroup": "advanced", + "type": "number", + "minimum": 0, + "title": "Max Session Length", + "longDescription": "The maximum allowed session length in days. For a session exceeding this length, all events after this limit will stop being processed. Exists to reduce lengthy table scans that can occur due to long sessions which are usually a result of bots.", + "packageDefault": "3", + "group": "Operation and Logic", + "description": "The maximum allowed session length in days. For a session exceeding this length, all events after this limit will stop being processed" + }, + "snowplow__package_name": { + "recommendFullRefresh": false, + "order": 113, + "consoleGroup": "advanced", + "type": "number", + "minimum": 0, + "title": "Package Name", + "group": "Operation and Logic", + "longDescription": "The name of the package you are running this macro under. This has implications for your `manifest` table.", + "packageDefault": "snowplow", + "description": "The name of the package you are running this macro under" + }, + "snowplow__session_lookback_days": { + "recommendFullRefresh": false, + "order": 121, + "consoleGroup": "advanced", + "type": "number", + "minimum": 0, + "title": "Session Lookback Window", + "longDescription": "Number of days to limit scan on `snowplow_web_base_sessions_lifecycle_manifest` manifest. Exists to improve performance of model when we have a lot of sessions. Should be set to as large a number as practical.", + "packageDefault": "730", + "group": "Operation and Logic", + "description": "Number of days to limit scan on `snowplow_web_base_sessions_lifecycle_manifest` manifest" + }, + "snowplow__upsert_lookback_days": { + "recommendFullRefresh": false, + "order": 126, + "consoleGroup": "advanced", + "type": "number", + "minimum": 0, + "title": "Upsert Lookback Days", + "group": "Operation and Logic", + "longDescription": "Number of days to look back over the incremental derived tables during the upsert. Where performance is not a concern, should be set to as long a value as possible. Having too short a period can result in duplicates. Please see the [Snowplow Optimized Materialization](docs/modeling-your-data/modeling-your-data-with-dbt/package-mechanics/optimized-upserts/) section for more details.", + "packageDefault": "30", + "description": "Number of days to look back over the incremental derived tables during the upsert" + }, + "snowplow__app_id": { + "recommendFullRefresh": false, + "order": 8, + "consoleGroup": "basic", + "type": "array", + "description": "> Click the plus sign to add a new entry", + "minItems": 0, + "title": "App IDs", + "longDescription": "A list of `app_id`s to filter the events table on for processing within the package.", + "packageDefault": "[ ] (no filter applied)", + "group": "Contexts, Filters, and Logs", + "items": { + "type": "string" + } + }, + "snowplow__session_identifiers": { + "recommendFullRefresh": true, + "order": 46, + "consoleGroup": "advanced", + "title": "Session Identifiers", + "group": "Operation and Logic", + "longDescription": "A list of key:value dictionaries which contain all of the contexts and fields where your session identifiers are located. For each entry in the list, if your map contains the `schema` value `atomic`, then this refers to a field found directly in the atomic `events` table. If you are trying to introduce a context/entity with an identifier in it, the package will look for the context in your events table with the name specified in the `schema` field. It will use the specified value in the `field` key as the field name to access. For Redshift/Postgres, using the `schema` key the package will try to find a table in your `snowplow__events_schema` schema with the same name as the `schema` value provided, and join that. If multiple fields are specified, the package will try to coalesce all fields in the order specified in the list. For a better understanding of the advanced usage of this variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details.", + "packageDefault": "[{\"schema\" : \"atomic\", \"field\" : \"domain_sessionid\"}]", + "type": "array", + "description": "> Click the plus sign to add a new entry", + "minItems": 0, + "items": { + "type": "object", + "title": "Identifier", + "properties": { + "schema": { + "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, + "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" + }, + "field": { + "type": "string", + "order": 2, + "title": "Field name", + "description": "The name of the field to use as session identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" + } + }, + "required": [ + "schema", + "field" + ], + "additionalProperties": false + }, + "uniqueItems": true + }, + "snowplow__session_sql": { + "recommendFullRefresh": true, + "order": 47, + "consoleGroup": "advanced", + "type": "string", + "title": "SQL for your session identifier", + "longDescription": "This allows you to override the `session_identifiers` SQL, to define completely custom SQL in order to build out a session identifier for your events. If you are interested in using this instead of providing identifiers through the `session_identifiers` variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details on how to do that.", + "packageDefault": "", + "group": "Operation and Logic" + }, + "snowplow__session_timestamp": { + "recommendFullRefresh": false, + "order": 55, + "consoleGroup": "advanced", + "type": "string", + "title": "Timestamp used for incremental processing, should be your partition field", + "group": "Operation and Logic", + "longDescription": "Determines which timestamp is used to build the sessionization logic. It's a good idea to have this timestamp be the same timestamp as the field you partition your events table on.", + "packageDefault": "collector_tstamp" + }, + "snowplow__user_identifiers": { + "recommendFullRefresh": true, + "order": 48, + "consoleGroup": "advanced", + "title": "User Identifiers", + "group": "Operation and Logic", + "longDescription": "A list of key:value dictionaries which contain all of the contexts and fields where your user identifiers are located. For each entry in the list, if your map contains the `schema` value `atomic`, then this refers to a field found directly in the atomic `events` table. If you are trying to introduce a context/entity with an identifier in it, the package will look for the context in your events table with the name specified in the `schema` field. It will use the specified value in the `field` key as the field name to access. For Redshift/Postgres, using the `schema` key the package will try to find a table in your `snowplow__events_schema` schema with the same name as the `schema` value provided, and join that. If multiple fields are specified, the package will try to coalesce all fields in the order specified in the list. For a better understanding of the advanced usage of this variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details.", + "packageDefault": "[{\"schema\" : \"atomic\", \"field\" : \"domain_userid\"}]", + "type": "array", + "description": "> Click the plus sign to add a new entry", + "minItems": 0, + "items": { + "type": "object", + "title": "Identifier", + "properties": { + "schema": { + "type": "string", + "title": "(JSON) schema name for the field", + "order": 1, + "description": "The schema name of your events table, atomic in most use cases, alternatively for sdes/contexts this should instead be the name of the field itself" + }, + "field": { + "type": "string", + "order": 2, + "title": "Field name", + "description": "The name of the field to use as user identifier, alternatively, in case of sdes/contexts it is the name of the element that refers to the field to be extracted" + } + }, + "required": [ + "schema", + "field" + ], + "additionalProperties": false + }, + "uniqueItems": true + }, + "snowplow__user_sql": { + "recommendFullRefresh": true, + "order": 49, + "consoleGroup": "advanced", + "type": "string", + "title": "SQL for your user identifier", + "longDescription": "This allows you to override the `user_identifiers` SQL, to define completely custom SQL in order to build out a user identifier for your events. If you are interested in using this instead of providing identifiers through the `user_identifiers` variable, please see the [Custom Identifiers](docs/modeling-your-data/modeling-your-data-with-dbt/package-features/customer-identifiers/) section for more details on how to do that.", + "packageDefault": "", + "group": "Operation and Logic" + }, + "snowplow__entities_or_sdes": { + "recommendFullRefresh": false, + "order": 104, + "consoleGroup": "advanced", + "title": "(Redshift) Entities or SDEs", + "longDescription": "A list of dictionaries defining the `context` or `self-describing` event tables to join onto your base events table. Please use the tool below or see the section on [Utilizing custom contexts or SDEs](/docs/modeling-your-data/modeling-your-data-with-dbt/package-features/modeling-entities/) for details of the structure.", + "packageDefault": "[]", + "warehouse": "Redshift", + "group": "Warehouse Specific", + "type": "array", + "description": "> Click the plus sign to add a new entry", + "minItems": 0, + "items": { + "type": "object", + "title": "Entity or SDE", + "properties": { + "schema": { + "type": "string", + "title": "Table name", + "description": "Table name", + "order": 1 + }, + "prefix": { + "type": "string", + "title": "Column prefix", + "description": "Prefix to add to columns", + "order": 2 + }, + "alias": { + "type": "string", + "title": "CTE Alias", + "description": "Table alias for the subquery", + "order": 3 + }, + "single_entity": { + "type": "boolean", + "title": "Is single entity?", + "order": 4 + } + }, + "required": [ + "schema", + "prefix" + ], + "additionalProperties": false + }, + "uniqueItems": true + } + } +}