Databricks support

.github/workflows/branch_replication.yml

@@ -0,0 +1,34 @@
+name: Replicate secondary master branches
+
+on:
+  workflow_dispatch:
+  # push:
+  #   branches:
+  #     - master
+
+jobs:
+  move-branch-pointer:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Git user
+        run: |
+          git config --global user.name "github-branch-replicator[bot]"
+          git config --global user.email "github-branch-replicator[bot]@users.noreply.github.com"
+
+      - name: Move branch pointer
+        run: |
+          # Fetch the latest changes
+          git fetch origin
+          # Force move branch 'master-databricks' to 'master'
+          git branch -f master-databricks master
+          # Push the changes to branch 'master-databricks'
+          git push -u origin master-databricks --force
+
+      - name: Verify move
+        run: |
+          git checkout master-databricks
+          git log -1

.github/workflows/pr_job.yml

@@ -3,14 +3,18 @@ name: dbt PR job
 on:
   pull_request:
     types:
-    - opened
-    - reopened
-    - synchronize
-    - ready_for_review
-  push:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
     branches:
-      - '!master'
+      - master
+  # push:
+  #   branches-ignore:
+  #     - master
+  #     - master-databricks
 
+
 jobs:
   run:
     runs-on: ubuntu-20.04
@@ -46,13 +50,13 @@ jobs:
           SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
           SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
           SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
-          SNOWFLAKE_SCHEMA: "${{ 'PR' }}"
-
+          # SNOWFLAKE_SCHEMA: "${{ 'PR' }}"
+          SNOWFLAKE_SCHEMA: "${{ format('{0}_{1}', 'PR_NUM', steps.findPr.outputs.pr) }}"
       - name: submit artifacts to datafold
         run: |
           set -ex
           datafold dbt upload --ci-config-id 146 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA}
         env:
-          DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }}
+          DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY }}
           DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}"
           GIT_SHA: "${{ github.event.pull_request.head.sha }}"

.github/workflows/pr_job_databricks.yml

@@ -0,0 +1,61 @@
+name: dbt PR job (Databricks)
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+    branches:
+      - master-databricks
+  # push:
+  #   branches-ignore:
+  #     - master
+  #     - master-databricks
+
+jobs:
+  run:
+    runs-on: ubuntu-20.04
+    if: ${{ !github.event.pull_request.draft }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+
+      - name: install requirements
+        run: pip install -q -r requirements.txt
+
+      - name: install datafold-sdk
+        run: pip install -q datafold-sdk
+
+      - name: dbt deps
+        run: dbt deps
+
+      - name: Find Current Pull Request
+        uses: jwalton/gh-find-current-pr@v1.3.0
+        id: findPR
+
+      - name: dbt build
+        run: dbt build --full-refresh --profiles-dir ./ --profile demo_databricks
+        env:
+          DATABRICKS_DEMO_HOST: ${{ secrets.DATABRICKS_DEMO_HOST }}
+          DATABRICKS_DEMO_PATH: ${{ secrets.DATABRICKS_DEMO_PATH }}
+          DATABRICKS_DEMO_TOKEN: ${{ secrets.DATABRICKS_DEMO_TOKEN }}
+          # DATABRICKS_DEMO_SCHEMA: "${{ 'PR' }}"
+          DATABRICKS_DEMO_SCHEMA: "${{ format('{0}_{1}', 'PR_NUM', steps.findPr.outputs.pr) }}"
+
+      - name: submit artifacts to datafold
+        run: |
+          set -ex
+          datafold dbt upload --ci-config-id 420 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA}
+        env:
+          DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY_DATABRICKS }}
+          DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}"
+          GIT_SHA: "${{ github.event.pull_request.head.sha }}"

.github/workflows/prod_job.yml

@@ -41,6 +41,6 @@ jobs:
           set -ex
           datafold dbt upload --ci-config-id 146 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA}
         env:
-          DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }}
+          DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY }}
           DATAFOLD_RUN_TYPE: "${{ github.ref == 'refs/heads/master' && 'production' || 'pull_request' }}"
           GIT_SHA: "${{ github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}"

.github/workflows/prod_job_databricks.yml

@@ -0,0 +1,45 @@
+name: dbt prod (Databricks)
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master-databricks
+
+jobs:
+  run:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+
+      - name: install requirements
+        run: pip install -q -r requirements.txt
+
+      - name: install datafold-sdk
+        run: pip install -q datafold-sdk
+
+      - name: dbt deps
+        run: dbt deps
+
+      - name: dbt build
+        run: dbt build --full-refresh --profiles-dir ./ --profile demo_databricks
+        env:
+          DATABRICKS_DEMO_HOST: ${{ secrets.DATABRICKS_DEMO_HOST }}
+          DATABRICKS_DEMO_PATH: ${{ secrets.DATABRICKS_DEMO_PATH }}
+          DATABRICKS_DEMO_TOKEN: ${{ secrets.DATABRICKS_DEMO_TOKEN }}
+          SNOWFLAKE_SCHEMA: "${{ 'default' }}"
+
+      - name: submit artifacts to datafold
+        run: |
+          set -ex
+          datafold dbt upload --ci-config-id 420 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA}
+        env:
+          DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY_DATABRICKS }}
+          DATAFOLD_RUN_TYPE: "${{ github.ref == 'refs/heads/master-databricks' && 'production' || 'pull_request' }}"
+          GIT_SHA: "${{ github.ref == 'refs/heads/master-databricks' && github.sha || github.event.pull_request.head.sha }}"

profiles.yml

@@ -11,3 +11,15 @@ demo:
       warehouse: INTEGRATION
       schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}"
       threads: 24
+
+demo_databricks:
+  target: dev
+  outputs:
+    dev:
+      type: databricks
+      catalog: demo
+      schema: "{{ env_var('DATABRICKS_DEMO_SCHEMA') | as_text }}"
+      host: "{{ env_var('DATABRICKS_DEMO_HOST') }}"
+      http_path: "{{ env_var('DATABRICKS_DEMO_PATH') }}"
+      token: "{{ env_var('DATABRICKS_DEMO_TOKEN') }}"
+      threads: 5

requirements.txt

@@ -1 +1,2 @@
-dbt-snowflake==1.6.2
+dbt-snowflake==1.8.3
+dbt-databricks==1.8.3