diff --git a/.github/workflows/branch_replication.yml b/.github/workflows/branch_replication.yml new file mode 100644 index 00000000..f9826ec4 --- /dev/null +++ b/.github/workflows/branch_replication.yml @@ -0,0 +1,34 @@ +name: Replicate secondary master branches + +on: + workflow_dispatch: + # push: + # branches: + # - master + +jobs: + move-branch-pointer: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Git user + run: | + git config --global user.name "github-branch-replicator[bot]" + git config --global user.email "github-branch-replicator[bot]@users.noreply.github.com" + + - name: Move branch pointer + run: | + # Fetch the latest changes + git fetch origin + # Force move branch 'master-databricks' to 'master' + git branch -f master-databricks master + # Push the changes to branch 'master-databricks' + git push -u origin master-databricks --force + + - name: Verify move + run: | + git checkout master-databricks + git log -1 diff --git a/.github/workflows/pr_job.yml b/.github/workflows/pr_job.yml index 096c8402..816c41fd 100644 --- a/.github/workflows/pr_job.yml +++ b/.github/workflows/pr_job.yml @@ -3,14 +3,18 @@ name: dbt PR job on: pull_request: types: - - opened - - reopened - - synchronize - - ready_for_review - push: + - opened + - reopened + - synchronize + - ready_for_review branches: - - '!master' + - master + # push: + # branches-ignore: + # - master + # - master-databricks + jobs: run: runs-on: ubuntu-20.04 @@ -46,13 +50,13 @@ jobs: SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} - SNOWFLAKE_SCHEMA: "${{ 'PR' }}" - + # SNOWFLAKE_SCHEMA: "${{ 'PR' }}" + SNOWFLAKE_SCHEMA: "${{ format('{0}_{1}', 'PR_NUM', steps.findPr.outputs.pr) }}" - name: submit artifacts to datafold run: | set -ex datafold dbt upload --ci-config-id 146 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} env: - DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }} + DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY }} DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}" GIT_SHA: "${{ github.event.pull_request.head.sha }}" diff --git a/.github/workflows/pr_job_databricks.yml b/.github/workflows/pr_job_databricks.yml new file mode 100644 index 00000000..3d0bb316 --- /dev/null +++ b/.github/workflows/pr_job_databricks.yml @@ -0,0 +1,61 @@ +name: dbt PR job (Databricks) + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + branches: + - master-databricks + # push: + # branches-ignore: + # - master + # - master-databricks + +jobs: + run: + runs-on: ubuntu-20.04 + if: ${{ !github.event.pull_request.draft }} + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - name: checkout + uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: install requirements + run: pip install -q -r requirements.txt + + - name: install datafold-sdk + run: pip install -q datafold-sdk + + - name: dbt deps + run: dbt deps + + - name: Find Current Pull Request + uses: jwalton/gh-find-current-pr@v1.3.0 + id: findPR + + - name: dbt build + run: dbt build --full-refresh --profiles-dir ./ --profile demo_databricks + env: + DATABRICKS_DEMO_HOST: ${{ secrets.DATABRICKS_DEMO_HOST }} + DATABRICKS_DEMO_PATH: ${{ secrets.DATABRICKS_DEMO_PATH }} + DATABRICKS_DEMO_TOKEN: ${{ secrets.DATABRICKS_DEMO_TOKEN }} + # DATABRICKS_DEMO_SCHEMA: "${{ 'PR' }}" + DATABRICKS_DEMO_SCHEMA: "${{ format('{0}_{1}', 'PR_NUM', steps.findPr.outputs.pr) }}" + + - name: submit artifacts to datafold + run: | + set -ex + datafold dbt upload --ci-config-id 420 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} + env: + DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY_DATABRICKS }} + DATAFOLD_RUN_TYPE: "${{ 'pull_request' }}" + GIT_SHA: "${{ github.event.pull_request.head.sha }}" diff --git a/.github/workflows/prod_job.yml b/.github/workflows/prod_job.yml index 0f8e0c9d..bfc4f887 100644 --- a/.github/workflows/prod_job.yml +++ b/.github/workflows/prod_job.yml @@ -41,6 +41,6 @@ jobs: set -ex datafold dbt upload --ci-config-id 146 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} env: - DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }} + DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY }} DATAFOLD_RUN_TYPE: "${{ github.ref == 'refs/heads/master' && 'production' || 'pull_request' }}" GIT_SHA: "${{ github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}" diff --git a/.github/workflows/prod_job_databricks.yml b/.github/workflows/prod_job_databricks.yml new file mode 100644 index 00000000..6bd46f35 --- /dev/null +++ b/.github/workflows/prod_job_databricks.yml @@ -0,0 +1,45 @@ +name: dbt prod (Databricks) + +on: + workflow_dispatch: + push: + branches: + - master-databricks + +jobs: + run: + runs-on: ubuntu-20.04 + + steps: + - name: checkout + uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: install requirements + run: pip install -q -r requirements.txt + + - name: install datafold-sdk + run: pip install -q datafold-sdk + + - name: dbt deps + run: dbt deps + + - name: dbt build + run: dbt build --full-refresh --profiles-dir ./ --profile demo_databricks + env: + DATABRICKS_DEMO_HOST: ${{ secrets.DATABRICKS_DEMO_HOST }} + DATABRICKS_DEMO_PATH: ${{ secrets.DATABRICKS_DEMO_PATH }} + DATABRICKS_DEMO_TOKEN: ${{ secrets.DATABRICKS_DEMO_TOKEN }} + SNOWFLAKE_SCHEMA: "${{ 'default' }}" + + - name: submit artifacts to datafold + run: | + set -ex + datafold dbt upload --ci-config-id 420 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} + env: + DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY_DATABRICKS }} + DATAFOLD_RUN_TYPE: "${{ github.ref == 'refs/heads/master-databricks' && 'production' || 'pull_request' }}" + GIT_SHA: "${{ github.ref == 'refs/heads/master-databricks' && github.sha || github.event.pull_request.head.sha }}" diff --git a/profiles.yml b/profiles.yml index 9d65d64f..9d91ae62 100644 --- a/profiles.yml +++ b/profiles.yml @@ -11,3 +11,15 @@ demo: warehouse: INTEGRATION schema: "{{ env_var('SNOWFLAKE_SCHEMA') | as_text }}" threads: 24 + +demo_databricks: + target: dev + outputs: + dev: + type: databricks + catalog: demo + schema: "{{ env_var('DATABRICKS_DEMO_SCHEMA') | as_text }}" + host: "{{ env_var('DATABRICKS_DEMO_HOST') }}" + http_path: "{{ env_var('DATABRICKS_DEMO_PATH') }}" + token: "{{ env_var('DATABRICKS_DEMO_TOKEN') }}" + threads: 5 diff --git a/requirements.txt b/requirements.txt index fc15ed4d..76ab4b59 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -dbt-snowflake==1.6.2 +dbt-snowflake==1.8.3 +dbt-databricks==1.8.3