Merge pull request #19 from fivetran/bug/add_fivetran_id

fivetran-catfritz · web-flow · commit 53843b7aece4 · 2023-03-08T14:43:48.000-06:00
bug/add_fivetran_id
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -69,5 +69,6 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,13 @@
+# dbt_pendo_source v0.4.0
+
+## 🚨 Breaking Changes 🚨:
+## 🔧 Bug Fixes
+- Updated models `stg_pendo__feature_event` and `stg_pendo__page_event` to include `_fivetran_id`, which was recently added per the Fivetran Pendo Connector's [release notes](https://fivetran.com/docs/applications/pendo/changelog#december2022). ([#19](https://github.com/fivetran/dbt_pendo_source/pull/19))
+
+## ✨ Features
+- Update documentation for `_fivetran_id`. ([#19](https://github.com/fivetran/dbt_pendo_source/pull/19))
+- Revised source yml and readme instructions for setting up the `GROUP` table with Snowflake. ([#19](https://github.com/fivetran/dbt_pendo_source/pull/19))
+
 # dbt_pendo_source v0.3.1
 ## Bug Fixes
 - Updated readme for workaround if the pendo_<default_source_table_name>_identifer is having trouble with Snowflake reserved words. ([#17](https://github.com/fivetran/dbt_pendo_source/pull/17))
diff --git a/README.md b/README.md
@@ -38,7 +38,7 @@ Include the following pendo_source package version in your `packages.yml` file.
 ```yaml
 packages:
   - package: fivetran/pendo_source
-    version: [">=0.3.0", "<0.4.0"]
+    version: [">=0.4.0", "<0.5.0"]
 ```
 
 ## Step 3: Define database and schema variables
@@ -99,17 +99,27 @@ vars:
   pendo_source:
     pendo_<default_source_table_name>_identifier: "your_table_name" 
 ```
-#### 🚨 Snowflake Users
+#### 🚨 Snowflake Users 🚨
 You may need to provide the case-sensitive spelling of your source tables that are also Snowflake reserved words.
 
 In this package, this would apply to the `GROUP` source. If you are receiving errors for this source, include the following in your `dbt_project.yml` file:
-
 ```yml
 vars:
-  pendo_group_identifier: '"Group"' # as an example, must include this quoting pattern and your exact casing
+  pendo_group_identifier: '"Group"' # as an example, must include this quoting pattern and adjust for your exact casing
+```
+
 
-quoting:
-  identifier: true
+**Note!** if you have sources defined in one of your project's yml files, for example if you have a yml file with a `sources` level like in the following example, the prior code will not work. 
+
+Instead you will need to add the following where your group source table is defined in your yml:
+```yml
+sources:
+  tables:
+    - name: group 
+      # Add the below
+      identifier: GROUP # Or what your group table is named, being mindful of casing
+      quoting:
+        identifier: true
 ```
 </details>
 
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,6 +1,6 @@
 config-version: 2
 name: 'pendo_source'
-version: '0.3.1'
+version: '0.4.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   pendo_source:
diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -45,7 +45,7 @@ integration_tests:
       schema: pendo_source_integration_tests_2000
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: pendo_source_integration_tests_2000
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'pendo_source_integration_tests'
-version: '0.3.1'
+version: '0.4.0'
 
 profile: 'integration_tests'
 
@@ -17,7 +17,8 @@ vars:
     pendo_event_identifier: "event"
     pendo_feature_event_identifier: "feature_event"
     pendo_feature_history_identifier: "feature_history"
-    pendo_group_identifier: "group_data"
+    group: "{{ ref('group_data') }}" # Comment out when generating docs; remember to uncomment before committing
+    # pendo_group_identifier: "group_data" # Uncomment when generating docs; remember to comment out before committing
     pendo_guide_event_identifier: "guide_event"
     pendo_guide_history_identifier: "guide_history"
     pendo_guide_step_history_identifier: "guide_step_history"
diff --git a/macros/get_feature_event_columns.sql b/macros/get_feature_event_columns.sql
@@ -11,7 +11,8 @@
     {"name": "server_name", "datatype": dbt.type_string()},
     {"name": "timestamp", "datatype": dbt.type_timestamp()},
     {"name": "user_agent", "datatype": dbt.type_string()},
-    {"name": "visitor_id", "datatype": dbt.type_string()}
+    {"name": "visitor_id", "datatype": dbt.type_string()},
+    {"name": "_fivetran_id", "datatype": dbt.type_string()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('pendo__feature_event_pass_through_columns')) }}
diff --git a/macros/get_page_event_columns.sql b/macros/get_page_event_columns.sql
@@ -11,7 +11,8 @@
     {"name": "server_name", "datatype": dbt.type_string()},
     {"name": "timestamp", "datatype": dbt.type_timestamp()},
     {"name": "user_agent", "datatype": dbt.type_string()},
-    {"name": "visitor_id", "datatype": dbt.type_string()}
+    {"name": "visitor_id", "datatype": dbt.type_string()},
+    {"name": "_fivetran_id", "datatype": dbt.type_string()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('pendo__page_event_pass_through_columns')) }}
diff --git a/models/src_pendo.yml b/models/src_pendo.yml
@@ -198,8 +198,10 @@ sources:
 
       - name: group
         identifier: >
-          {%- if target.type == 'snowflake' -%}{{ var('pendo_group_identifier', '"GROUP"') }}
+          {%- if target.type == 'snowflake' -%}{{ var('pendo_group_identifier', 'GROUP') }}
           {%- else -%}{{ var('pendo_group_identifier', 'group') }}{%- endif -%}
+        quoting:
+          identifier: true
         description: Table capturing the product areas.
         columns:
           - name: app_id
diff --git a/models/stg_pendo.yml b/models/stg_pendo.yml
@@ -132,8 +132,10 @@ models:
         description: String identifying the browser and operating system on which the event was triggered.
       - name: visitor_id
         description: Foreign key referencing the `VISITOR` who triggered the event. 
+      - name: _fivetran_id
+        description: Fivetran created id hashed on the `property_*` and `param_*` fields.
       - name: feature_event_key
-        description: Unique key hashed on ['visitor_id', 'timestamp', 'account_id', 'server_name', 'feature_id', 'remote_ip', 'user_agent'].
+        description: Unique key hashed on ['visitor_id', 'timestamp', 'account_id', 'server_name', 'feature_id', 'remote_ip', 'user_agent', '_fivetran_id'].
         tests:
           - unique
           - not_null
@@ -339,8 +341,10 @@ models:
         description: String identifying the browser and operating system on which the event was triggered.
       - name: visitor_id
         description: Foreign key referencing the `VISITOR` who triggered the event. 
+      - name: _fivetran_id
+        description: Fivetran created id hashed on the `param_*` fields.
       - name: page_event_key
-        description: Unique key hashed on ['visitor_id', 'timestamp', 'account_id', 'server_name', 'page_id', 'user_agent', 'remote_ip'].
+        description: Unique key hashed on ['visitor_id', 'timestamp', 'account_id', 'server_name', 'page_id', 'user_agent', 'remote_ip', '_fivetran_id'].
         tests:
           - unique
           - not_null
diff --git a/models/stg_pendo__feature_event.sql b/models/stg_pendo__feature_event.sql
@@ -33,7 +33,10 @@ final as (
         user_agent,
         visitor_id,
         _fivetran_synced,
-        {{ dbt_utils.generate_surrogate_key(['visitor_id', 'timestamp', 'account_id', 'server_name', 'feature_id', 'remote_ip', 'user_agent']) }} as feature_event_key
+        _fivetran_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['visitor_id', 'timestamp', 'account_id', 'server_name', 'feature_id', 'remote_ip', 'user_agent', '_fivetran_id']
+            ) }} as feature_event_key
 
         --The below macro adds the fields defined within your pendo__feature_event_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('pendo__feature_event_pass_through_columns') }}
diff --git a/models/stg_pendo__page_event.sql b/models/stg_pendo__page_event.sql
@@ -34,7 +34,10 @@ final as (
         user_agent,
         visitor_id,
         _fivetran_synced,
-        {{ dbt_utils.generate_surrogate_key(['visitor_id', 'timestamp', 'account_id', 'server_name', 'page_id', 'user_agent', 'remote_ip']) }} as page_event_key
+        _fivetran_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['visitor_id', 'timestamp', 'account_id', 'server_name', 'page_id', 'user_agent', 'remote_ip', '_fivetran_id']
+            ) }} as page_event_key
 
         --The below macro adds the fields defined within your pendo__page_event_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('pendo__page_event_pass_through_columns') }}
