dbt prod (Databricks) #365
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: dbt prod (Databricks) | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - master-databricks | |
| schedule: | |
| - cron: '20 0 * * *' | |
| jobs: | |
| run: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: checkout master-databricks | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: master-databricks | |
| - name: Verify and set branch and SHA | |
| run: | | |
| # echo "Checking if the current branch is master-databricks" | |
| # CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| # if [ "$CURRENT_BRANCH" != "master-databricks" ]; then | |
| # echo "This workflow runs only on master-databricks." | |
| # exit 1 | |
| # fi | |
| echo "GH_REF=$(git symbolic-ref HEAD)" >> $GITHUB_ENV | |
| echo "GH_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV | |
| - name: Verify environment variables | |
| run: | | |
| echo "GH_REF is $GH_REF" | |
| echo "GH_SHA is $GH_SHA" | |
| env: | |
| GH_REF: "${{ env.GH_REF }}" | |
| GH_SHA: "${{ env.GH_SHA }}" | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.8' | |
| - name: install requirements | |
| run: | | |
| echo "setuptools<72" > constraints.txt | |
| export PIP_CONSTRAINT=constraints.txt | |
| pip install --upgrade pip | |
| pip install -q -r requirements.txt | |
| - name: install datafold-sdk | |
| run: pip install -q datafold-sdk | |
| - name: dbt deps | |
| run: dbt deps | |
| - name: dbt build | |
| run: dbt build --full-refresh --profiles-dir ./ --profile demo_databricks | |
| env: | |
| DATABRICKS_DEMO_HOST: ${{ secrets.DATABRICKS_DEMO_HOST }} | |
| DATABRICKS_DEMO_PATH: ${{ secrets.DATABRICKS_DEMO_PATH }} | |
| DATABRICKS_DEMO_TOKEN: ${{ secrets.DATABRICKS_DEMO_TOKEN }} | |
| DATABRICKS_DEMO_SCHEMA: "${{ 'default' }}" | |
| - name: submit artifacts to datafold | |
| run: | | |
| set -ex | |
| datafold dbt upload --ci-config-id 420 --run-type ${DATAFOLD_RUN_TYPE} --commit-sha ${GIT_SHA} | |
| env: | |
| DATAFOLD_API_KEY: ${{ secrets.DATAFOLD_APIKEY }} | |
| DATAFOLD_RUN_TYPE: "${{ env.GH_REF == 'refs/heads/master-databricks' && 'production' || 'pull_request' }}" | |
| GIT_SHA: "${{ env.GH_REF == 'refs/heads/master-databricks' && env.GH_SHA || github.event.pull_request.head.sha }}" |