diff --git a/.changes/0.0.0.md b/.changes/0.0.0.md deleted file mode 100644 index 0bec014d9..000000000 --- a/.changes/0.0.0.md +++ /dev/null @@ -1,12 +0,0 @@ -## Previous Releases -For information on prior major and minor releases, see their changelogs: -- [1.6](https://github.com/dbt-labs/dbt-spark/blob/1.6.latest/CHANGELOG.md) -- [1.5](https://github.com/dbt-labs/dbt-spark/blob/1.5.latest/CHANGELOG.md) -- [1.4](https://github.com/dbt-labs/dbt-spark/blob/1.4.latest/CHANGELOG.md) -- [1.3](https://github.com/dbt-labs/dbt-spark/blob/1.3.latest/CHANGELOG.md) -- [1.2](https://github.com/dbt-labs/dbt-spark/blob/1.2.latest/CHANGELOG.md) -- [1.1](https://github.com/dbt-labs/dbt-spark/blob/1.1.latest/CHANGELOG.md) -- [1.0](https://github.com/dbt-labs/dbt-spark/blob/1.0.latest/CHANGELOG.md) -- [0.21](https://github.com/dbt-labs/dbt-spark/blob/0.21.latest/CHANGELOG.md) -- [0.20](https://github.com/dbt-labs/dbt-spark/blob/0.20.latest/CHANGELOG.md) -- [0.19 and earlier](https://github.com/dbt-labs/dbt-spark/blob/0.19.latest/CHANGELOG.md) diff --git a/.changes/README.md b/.changes/README.md deleted file mode 100644 index dc6106dfe..000000000 --- a/.changes/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# CHANGELOG - -To view information about the changelog operation we suggest reading this [README](https://github.com/dbt-labs/dbt-spark/blob/main/.changes/README.md) found in `dbt-spark`. diff --git a/.changes/header.tpl.md b/.changes/header.tpl.md deleted file mode 100644 index 251ea5d51..000000000 --- a/.changes/header.tpl.md +++ /dev/null @@ -1,6 +0,0 @@ -# dbt-spark Changelog - -- This file provides a full account of all changes to `dbt-spark`. -- Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. -- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. -- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-spark/blob/main/CONTRIBUTING.md#adding-changelog-entry) diff --git a/.changes/unreleased/.gitkeep b/.changes/unreleased/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/.changes/unreleased/Under the Hood-20241207-181814.yaml b/.changes/unreleased/Under the Hood-20241207-181814.yaml deleted file mode 100644 index c76974d43..000000000 --- a/.changes/unreleased/Under the Hood-20241207-181814.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Move from setup.py to pyproject.toml and to hatch as a dev tool -time: 2024-12-07T18:18:14.85697-05:00 -custom: - Author: mikealfare - Issue: "1150" diff --git a/.changie.yaml b/.changie.yaml deleted file mode 100644 index f4d5bdb96..000000000 --- a/.changie.yaml +++ /dev/null @@ -1,131 +0,0 @@ -changesDir: .changes -unreleasedDir: unreleased -headerPath: header.tpl.md -versionHeaderPath: "" -changelogPath: CHANGELOG.md -versionExt: md -envPrefix: "CHANGIE_" -versionFormat: '## dbt-spark {{.Version}} - {{.Time.Format "January 02, 2006"}}' -kindFormat: '### {{.Kind}}' -changeFormat: |- - {{- $IssueList := list }} - {{- $changes := splitList " " $.Custom.Issue }} - {{- range $issueNbr := $changes }} - {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-spark/issues/nbr)" | replace "nbr" $issueNbr }} - {{- $IssueList = append $IssueList $changeLink }} - {{- end -}} - - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}}) - -kinds: -- label: Breaking Changes -- label: Features -- label: Fixes -- label: Under the Hood -- label: Dependencies - changeFormat: |- - {{- $PRList := list }} - {{- $changes := splitList " " $.Custom.PR }} - {{- range $pullrequest := $changes }} - {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-spark/pull/nbr)" | replace "nbr" $pullrequest }} - {{- $PRList = append $PRList $changeLink }} - {{- end -}} - - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) - skipGlobalChoices: true - additionalChoices: - - key: Author - label: GitHub Username(s) (separated by a single space if multiple) - type: string - minLength: 3 - - key: PR - label: GitHub Pull Request Number (separated by a single space if multiple) - type: string - minLength: 1 -- label: Security - changeFormat: |- - {{- $PRList := list }} - {{- $changes := splitList " " $.Custom.PR }} - {{- range $pullrequest := $changes }} - {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-spark/pull/nbr)" | replace "nbr" $pullrequest }} - {{- $PRList = append $PRList $changeLink }} - {{- end -}} - - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) - skipGlobalChoices: true - additionalChoices: - - key: Author - label: GitHub Username(s) (separated by a single space if multiple) - type: string - minLength: 3 - - key: PR - label: GitHub Pull Request Number (separated by a single space if multiple) - type: string - minLength: 1 - -newlines: - afterChangelogHeader: 1 - afterKind: 1 - afterChangelogVersion: 1 - beforeKind: 1 - endOfVersion: 1 - -custom: -- key: Author - label: GitHub Username(s) (separated by a single space if multiple) - type: string - minLength: 3 -- key: Issue - label: GitHub Issue Number (separated by a single space if multiple) - type: string - minLength: 1 - -footerFormat: | - {{- $contributorDict := dict }} - {{- /* ensure all names in this list are all lowercase for later matching purposes */}} - {{- $core_team := splitList " " .Env.CORE_TEAM }} - {{- /* ensure we always skip snyk and dependabot in addition to the core team */}} - {{- $maintainers := list "dependabot[bot]" "snyk-bot"}} - {{- range $team_member := $core_team }} - {{- $team_member_lower := lower $team_member }} - {{- $maintainers = append $maintainers $team_member_lower }} - {{- end }} - {{- range $change := .Changes }} - {{- $authorList := splitList " " $change.Custom.Author }} - {{- /* loop through all authors for a single changelog */}} - {{- range $author := $authorList }} - {{- $authorLower := lower $author }} - {{- /* we only want to include non-core team contributors */}} - {{- if not (has $authorLower $maintainers)}} - {{- $changeList := splitList " " $change.Custom.Author }} - {{- $IssueList := list }} - {{- $changeLink := $change.Kind }} - {{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }} - {{- $changes := splitList " " $change.Custom.PR }} - {{- range $issueNbr := $changes }} - {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-spark/pull/nbr)" | replace "nbr" $issueNbr }} - {{- $IssueList = append $IssueList $changeLink }} - {{- end -}} - {{- else }} - {{- $changes := splitList " " $change.Custom.Issue }} - {{- range $issueNbr := $changes }} - {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-spark/issues/nbr)" | replace "nbr" $issueNbr }} - {{- $IssueList = append $IssueList $changeLink }} - {{- end -}} - {{- end }} - {{- /* check if this contributor has other changes associated with them already */}} - {{- if hasKey $contributorDict $author }} - {{- $contributionList := get $contributorDict $author }} - {{- $contributionList = concat $contributionList $IssueList }} - {{- $contributorDict := set $contributorDict $author $contributionList }} - {{- else }} - {{- $contributionList := $IssueList }} - {{- $contributorDict := set $contributorDict $author $contributionList }} - {{- end }} - {{- end}} - {{- end}} - {{- end }} - {{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}} - {{- if $contributorDict}} - ### Contributors - {{- range $k,$v := $contributorDict }} - - [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}}) - {{- end }} - {{- end }} diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml deleted file mode 100644 index f5494b313..000000000 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: 🐞 Bug -description: Report a bug or an issue you've found with dbt-spark -title: "[Bug] " -labels: ["bug", "triage"] -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this bug report! - - type: checkboxes - attributes: - label: Is this a new bug in dbt-spark? - description: > - In other words, is this an error, flaw, failure or fault in our software? - - If this is a bug that broke existing functionality that used to work, please open a regression issue. - If this is a bug in the dbt-core logic, please open an issue in the dbt-core repository. - If this is a bug experienced while using dbt Cloud, please report to [support](mailto:support@getdbt.com). - If this is a request for help or troubleshooting code in your own dbt project, please join our [dbt Community Slack](https://www.getdbt.com/community/join-the-community/) or open a [Discussion question](https://github.com/dbt-labs/docs.getdbt.com/discussions). - - Please search to see if an issue already exists for the bug you encountered. - options: - - label: I believe this is a new bug in dbt-spark - required: true - - label: I have searched the existing issues, and I could not find an existing issue for this bug - required: true - - type: textarea - attributes: - label: Current Behavior - description: A concise description of what you're experiencing. - validations: - required: true - - type: textarea - attributes: - label: Expected Behavior - description: A concise description of what you expected to happen. - validations: - required: true - - type: textarea - attributes: - label: Steps To Reproduce - description: Steps to reproduce the behavior. - placeholder: | - 1. In this environment... - 2. With this config... - 3. Run '...' - 4. See error... - validations: - required: true - - type: textarea - id: logs - attributes: - label: Relevant log output - description: | - If applicable, log output to help explain your problem. - render: shell - validations: - required: false - - type: textarea - attributes: - label: Environment - description: | - examples: - - **OS**: Ubuntu 20.04 - - **Python**: 3.9.12 (`python3 --version`) - - **dbt-core**: 1.1.1 (`dbt --version`) - - **dbt-spark**: 1.1.0 (`dbt --version`) - value: | - - OS: - - Python: - - dbt-core: - - dbt-spark: - render: markdown - validations: - required: false - - type: textarea - attributes: - label: Additional Context - description: | - Links? References? Anything that will give us more context about the issue you are encountering! - - Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 129ea7779..9f1294ffe 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -7,8 +7,11 @@ contact_links: url: mailto:support@getdbt.com about: Are you using dbt Cloud? Contact our support team for help! - name: Participate in Discussions - url: https://github.com/dbt-labs/dbt-spark/discussions + url: https://github.com/dbt-labs/dbt-adapters/discussions about: Do you have a Big Idea for dbt-spark? Read open discussions, or start a new one + - name: Create an issue for dbt-spark + url: https://github.com/dbt-labs/dbt-adapters/issues/new/choose + about: Report a bug or request a feature for dbt-spark - name: Create an issue for dbt-core url: https://github.com/dbt-labs/dbt-core/issues/new/choose about: Report a bug or request a feature for dbt-core diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml deleted file mode 100644 index 8c123ba51..000000000 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: ✨ Feature -description: Propose a straightforward extension of dbt-spark functionality -title: "[Feature] <title>" -labels: ["enhancement", "triage"] -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this feature request! - - type: checkboxes - attributes: - label: Is this your first time submitting a feature request? - description: > - We want to make sure that features are distinct and discoverable, - so that other members of the community can find them and offer their thoughts. - - Issues are the right place to request straightforward extensions of existing dbt-spark functionality. - For "big ideas" about future capabilities of dbt-spark, we ask that you open a - [discussion](https://github.com/dbt-labs/dbt-spark/discussions) in the "Ideas" category instead. - options: - - label: I have read the [expectations for open source contributors](https://docs.getdbt.com/docs/contributing/oss-expectations) - required: true - - label: I have searched the existing issues, and I could not find an existing issue for this feature - required: true - - label: I am requesting a straightforward extension of existing dbt-spark functionality, rather than a Big Idea better suited to a discussion - required: true - - type: textarea - attributes: - label: Describe the feature - description: A clear and concise description of what you want to happen. - validations: - required: true - - type: textarea - attributes: - label: Describe alternatives you've considered - description: | - A clear and concise description of any alternative solutions or features you've considered. - validations: - required: false - - type: textarea - attributes: - label: Who will this benefit? - description: | - What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly. - validations: - required: false - - type: input - attributes: - label: Are you interested in contributing this feature? - description: Let us know if you want to write some code, and how we can help. - validations: - required: false - - type: textarea - attributes: - label: Anything else? - description: | - Links? References? Anything that will give us more context about the feature you are suggesting! - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml deleted file mode 100644 index 8b65d6a26..000000000 --- a/.github/ISSUE_TEMPLATE/regression-report.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: ☣️ Regression -description: Report a regression you've observed in a newer version of dbt-spark -title: "[Regression] <title>" -labels: ["bug", "regression", "triage"] -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this regression report! - - type: checkboxes - attributes: - label: Is this a regression in a recent version of dbt-spark? - description: > - A regression is when documented functionality works as expected in an older version of dbt-spark, - and no longer works after upgrading to a newer version of dbt-spark - options: - - label: I believe this is a regression in dbt-spark functionality - required: true - - label: I have searched the existing issues, and I could not find an existing issue for this regression - required: true - - type: textarea - attributes: - label: Current Behavior - description: A concise description of what you're experiencing. - validations: - required: true - - type: textarea - attributes: - label: Expected/Previous Behavior - description: A concise description of what you expected to happen. - validations: - required: true - - type: textarea - attributes: - label: Steps To Reproduce - description: Steps to reproduce the behavior. - placeholder: | - 1. In this environment... - 2. With this config... - 3. Run '...' - 4. See error... - validations: - required: true - - type: textarea - id: logs - attributes: - label: Relevant log output - description: | - If applicable, log output to help explain your problem. - render: shell - validations: - required: false - - type: textarea - attributes: - label: Environment - description: | - examples: - - **OS**: Ubuntu 20.04 - - **Python**: 3.9.12 (`python3 --version`) - - **dbt-core (working version)**: 1.1.1 (`dbt --version`) - - **dbt-spark (working version)**: 1.1.0 (`dbt --version`) - - **dbt-core (regression version)**: 1.2.0 (`dbt --version`) - - **dbt-spark (regression version)**: 1.2.0 (`dbt --version`) - value: | - - OS: - - Python: - - dbt-core (working version): - - dbt-spark (working version): - - dbt-core (regression version): - - dbt-spark (regression version): - render: markdown - validations: - required: true - - type: textarea - attributes: - label: Additional Context - description: | - Links? References? Anything that will give us more context about the issue you are encountering! - - Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. - validations: - required: false diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 746dcae22..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "pip" - directory: "/" - schedule: - interval: "daily" - rebase-strategy: "disabled" - ignore: - - dependency-name: "*" - update-types: - - version-update:semver-patch - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - rebase-strategy: "disabled" - ignore: - - dependency-name: "*" - update-types: - - version-update:semver-patch - - package-ecosystem: "docker" - directory: "/docker" - schedule: - interval: "weekly" - rebase-strategy: "disabled" - ignore: - - dependency-name: "*" - update-types: - - version-update:semver-patch diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index a3c340cc3..000000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,35 +0,0 @@ -resolves # -[docs](https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose) dbt-labs/docs.getdbt.com/# - -<!--- - Include the number of the issue addressed by this PR above if applicable. - PRs for code changes without an associated issue *will not be merged*. - See CONTRIBUTING.md for more information. - - Include the number of the docs issue that was opened for this PR. If - this change has no user-facing implications, "N/A" suffices instead. New - docs tickets can be created by clicking the link above or by going to - https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose. ---> - -### Problem - -<!--- - Describe the problem this PR is solving. What is the application state - before this PR is merged? ---> - -### Solution - -<!--- - Describe the way this PR solves the above problem. Add as much detail as you - can to help reviewers understand your changes. Include any alternatives and - tradeoffs you considered. ---> - -### Checklist - -- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md) and understand what's expected of me -- [ ] I have run this code in development and it appears to resolve the stated issue -- [ ] This PR includes tests, or tests are not required/relevant for this PR -- [ ] This PR has no interface changes (e.g. macros, cli, logs, json artifacts, config files, adapter interface, etc) or this PR has already received feedback and approval from Product or DX diff --git a/.github/workflows/bot-changelog.yml b/.github/workflows/bot-changelog.yml deleted file mode 100644 index 89972070e..000000000 --- a/.github/workflows/bot-changelog.yml +++ /dev/null @@ -1,60 +0,0 @@ -# **what?** -# When bots create a PR, this action will add a corresponding changie yaml file to that -# PR when a specific label is added. -# -# The file is created off a template: -# -# kind: <per action matrix> -# body: <PR title> -# time: <current timestamp> -# custom: -# Author: <PR User Login (generally the bot)> -# PR: <PR number> -# -# **why?** -# Automate changelog generation for more visability with automated bot PRs. -# -# **when?** -# Once a PR is created, label should be added to PR before or after creation. You can also -# manually trigger this by adding the appropriate label at any time. -# -# **how to add another bot?** -# Add the label and changie kind to the include matrix. That's it! -# - -name: Bot Changelog - -on: - pull_request: - # catch when the PR is opened with the label or when the label is added - types: [labeled] - -permissions: - contents: write - pull-requests: read - -jobs: - generate_changelog: - strategy: - matrix: - include: - - label: "dependencies" - changie_kind: "Dependencies" - - label: "snyk" - changie_kind: "Security" - runs-on: ubuntu-latest - - steps: - - - name: Create and commit changelog on bot PR - if: ${{ contains(github.event.pull_request.labels.*.name, matrix.label) }} - id: bot_changelog - uses: emmyoop/changie_bot@v1 - with: - GITHUB_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} - commit_author_name: "Github Build Bot" - commit_author_email: "<buildbot@fishtownanalytics.com>" - commit_message: "Add automated changelog yaml from template for bot PR" - changie_kind: ${{ matrix.changie_kind }} - label: ${{ matrix.label }} - custom_changelog_string: "custom:\n Author: ${{ github.event.pull_request.user.login }}\n PR: ${{ github.event.pull_request.number }}" diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/changelog-existence.yml deleted file mode 100644 index 6e51e8afc..000000000 --- a/.github/workflows/changelog-existence.yml +++ /dev/null @@ -1,41 +0,0 @@ -# **what?** -# Checks that a file has been committed under the /.changes directory -# as a new CHANGELOG entry. Cannot check for a specific filename as -# it is dynamically generated by change type and timestamp. -# This workflow should not require any secrets since it runs for PRs -# from forked repos. -# By default, secrets are not passed to workflows running from -# a forked repo. - -# **why?** -# Ensure code change gets reflected in the CHANGELOG. - -# **when?** -# This will run for all PRs going into main and *.latest. It will -# run when they are opened, reopened, when any label is added or removed -# and when new code is pushed to the branch. The action will then get -# skipped if the 'Skip Changelog' label is present is any of the labels. - -name: Check Changelog Entry - -on: - pull_request: - types: [opened, reopened, labeled, unlabeled, synchronize] - workflow_dispatch: - -defaults: - run: - shell: bash - -permissions: - contents: read - pull-requests: write - - -jobs: - changelog: - uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main - with: - changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see the [dbt-spark contributing guide](https://github.com/dbt-labs/dbt-spark/blob/main/CONTRIBUTING.MD).' - skip_label: 'Skip Changelog' - secrets: inherit # this is only acceptable because we own the action we're calling diff --git a/.github/workflows/cut-release-branch.yml b/.github/workflows/cut-release-branch.yml deleted file mode 100644 index f8dfa2173..000000000 --- a/.github/workflows/cut-release-branch.yml +++ /dev/null @@ -1,42 +0,0 @@ -# **what?** -# Calls a centralize3d workflow that will: -# 1. Cut a new branch (generally `*.latest`) -# 2. Also cleans up all files in `.changes/unreleased` and `.changes/previous version on -# `main` and bumps `main` to the input version. - -# **why?** -# Generally reduces the workload of engineers and reduces error. Allow automation. - -# **when?** -# This will run when called manually. - -name: Cut new release branch - -on: - workflow_dispatch: - inputs: - version_to_bump_main: - description: 'The alpha version main should bump to (ex. 1.6.0a1)' - required: true - new_branch_name: - description: 'The full name of the new branch (ex. 1.5.latest)' - required: true - -defaults: - run: - shell: bash - -permissions: - contents: write - -jobs: - cut_branch: - name: "Cut branch and clean up main for dbt-spark" - uses: dbt-labs/actions/.github/workflows/cut-release-branch.yml@main - with: - version_to_bump_main: ${{ inputs.version_to_bump_main }} - new_branch_name: ${{ inputs.new_branch_name }} - PR_title: "Cleanup main after cutting new ${{ inputs.new_branch_name }} branch" - PR_body: "This PR will fail CI until the dbt-core PR has been merged due to release version conflicts." - secrets: - FISHTOWN_BOT_PAT: ${{ secrets.FISHTOWN_BOT_PAT }} diff --git a/.github/workflows/docs-issues.yml b/.github/workflows/docs-issues.yml deleted file mode 100644 index f49cf517c..000000000 --- a/.github/workflows/docs-issues.yml +++ /dev/null @@ -1,41 +0,0 @@ -# **what?** -# Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed - -# **why?** -# To reduce barriers for keeping docs up to date - -# **when?** -# When an issue is labeled `user docs` and is closed as completed. Can be labeled before or after the issue is closed. - - -name: Open issues in docs.getdbt.com repo when an issue is labeled -run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}" - -on: - issues: - types: [labeled, closed] - -defaults: - run: - shell: bash - -permissions: - issues: write # comments on issues - -jobs: - open_issues: - # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. - # If this logic does not exist in this workflow, it runs the - # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having - # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we - # decide if it should run or not. - if: | - (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( - (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || - (github.event.action == 'labeled' && github.event.label.name == 'user docs')) - uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main - with: - issue_repository: "dbt-labs/docs.getdbt.com" - issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}" - issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." - secrets: inherit diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 458048a7f..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,165 +0,0 @@ -# **what?** -# Runs code quality checks, unit tests, and verifies python build on -# all code commited to the repository. This workflow should not -# require any secrets since it runs for PRs from forked repos. -# By default, secrets are not passed to workflows running from -# a forked repo. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to a release -# branch, and when manually triggered. - -name: Tests and Code Checks - -on: - push: - branches: - - "main" - - "*.latest" - pull_request: - workflow_dispatch: - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - code-quality: - name: code-quality - - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - name: Check out the repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - name: Run pre-commit hooks - uses: pre-commit/action@v3.0.1 - - unit: - name: unit test / python ${{ matrix.python-version }} - - runs-on: ubuntu-latest - timeout-minutes: 10 - - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - - name: Check out the repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - uses: pypa/hatch@install - - - run: hatch run unit-tests - - build: - name: build packages - - runs-on: ubuntu-latest - - outputs: - is_alpha: ${{ steps.check-is-alpha.outputs.is_alpha }} - - steps: - - name: Check out the repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - uses: pypa/hatch@install - - - name: Build distributions - run: hatch build - - - name: Show distributions - run: ls -lh dist/ - - - name: Check distribution descriptions - run: hatch run build:check-all - - - name: Check if this is an alpha version - id: check-is-alpha - run: | - export is_alpha=0 - if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi - echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist/ - overwrite: true - - test-build: - name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }} - - if: needs.build.outputs.is_alpha == 0 - - needs: build - - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-14, windows-latest] - python-version: ["3.9", "3.10", "3.11", "3.12"] - dist-type: ["whl", "gz"] - - steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel - python -m pip --version - - - uses: actions/download-artifact@v4 - with: - name: dist - path: dist/ - - - name: Show distributions - run: ls -lh dist/ - - - name: Install ${{ matrix.dist-type }} distributions - run: | - find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check ${{ matrix.dist-type }} distributions - run: | - python -c "import dbt.adapters.spark" diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml deleted file mode 100644 index c467d3fce..000000000 --- a/.github/workflows/release-internal.yml +++ /dev/null @@ -1,127 +0,0 @@ -# What? -# -# Tag and release an arbitrary ref. Uploads to an internal archive for further processing. -# -# How? -# -# After checking out and testing the provided ref, the image is built and uploaded. -# -# When? -# -# Manual trigger. - -name: "Release to Cloud" -run-name: "Release to Cloud off of ${{ inputs.ref }}" - -on: - workflow_dispatch: - inputs: - ref: - description: "The ref (sha or branch name) to use" - type: string - default: "main" - required: true - package_test_command: - description: "Package test command" - type: string - default: "python -c \"import dbt.adapters.spark\"" - required: true - skip_tests: - description: "Should the tests be skipped? (default to false)" - type: boolean - required: true - default: false - -defaults: - run: - shell: "bash" - -env: - PYTHON_TARGET_VERSION: 3.9 - -jobs: - run-unit-tests: - name: "Unit tests" - - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - name: "Check out the repository" - uses: actions/checkout@v4 - - - name: "Set up Python ${{ env.PYTHON_TARGET_VERSION }}" - uses: actions/setup-python@v5 - with: - python-version: "${{ env.PYTHON_TARGET_VERSION }}" - - - name: Install python dependencies - uses: pypa/hatch@install - - - run: hatch run unit-tests - - run-integration-tests: - name: "${{ matrix.test }}" - needs: [run-unit-tests] - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - test: - - "apache_spark" - - "spark_session" - - "spark_http_odbc" - - "databricks_sql_endpoint" - - "databricks_cluster" - - "databricks_http_cluster" - - env: - DBT_INVOCATION_ENV: github-actions - DD_CIVISIBILITY_AGENTLESS_ENABLED: true - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_SITE: datadoghq.com - DD_ENV: ci - DD_SERVICE: ${{ github.event.repository.name }} - DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }} - DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} - DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} - DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} - DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }} - DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" - DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" - DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com" - - steps: - - name: Check out the repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - # the python version used here is not what is used in the tests themselves - - name: Set up Python for dagger - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install python dependencies - run: | - python -m pip install --user --upgrade pip - python -m pip --version - python -m pip install -r dagger/requirements.txt - - - name: "Run tests for ${{ matrix.test }}" - run: python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }} - - invoke-reusable-workflow: - name: "Create cloud release" - needs: [run-integration-tests] - uses: "dbt-labs/dbt-release/.github/workflows/internal-archive-release.yml@main" - - with: - package_test_command: "${{ inputs.package_test_command }}" - dbms_name: "spark" - ref: "${{ inputs.ref }}" - skip_tests: "true" - - secrets: "inherit" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index d902340a9..000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: "Close stale issues and PRs" -on: - schedule: - - cron: "30 1 * * *" - -permissions: - issues: write - pull-requests: write - -jobs: - stale: - uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/triage-labels.yml b/.github/workflows/triage-labels.yml deleted file mode 100644 index 91f529e3e..000000000 --- a/.github/workflows/triage-labels.yml +++ /dev/null @@ -1,31 +0,0 @@ -# **what?** -# When the core team triages, we sometimes need more information from the issue creator. In -# those cases we remove the `triage` label and add the `awaiting_response` label. Once we -# recieve a response in the form of a comment, we want the `awaiting_response` label removed -# in favor of the `triage` label so we are aware that the issue needs action. - -# **why?** -# To help with out team triage issue tracking - -# **when?** -# This will run when a comment is added to an issue and that issue has to `awaiting_response` label. - -name: Update Triage Label - -on: issue_comment - -defaults: - run: - shell: bash - -permissions: - issues: write - -jobs: - triage_label: - if: contains(github.event.issue.labels.*.name, 'awaiting_response') - uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main - with: - add_label: "triage" - remove_label: "awaiting_response" - secrets: inherit # this is only acceptable because we own the action we're calling diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 903507b7a..18b947966 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,128 +1,8 @@ # Contributing to `dbt-spark` -1. [About this document](#about-this-document) -3. [Getting the code](#getting-the-code) -5. [Running `dbt-spark` in development](#running-dbt-spark-in-development) -6. [Testing](#testing) -7. [Updating Docs](#updating-docs) -7. [Submitting a Pull Request](#submitting-a-pull-request) +This repository has moved into the `dbt-labs/dbt-adapters` monorepo found +[here](https://www.github.com/dbt-labs/dbt-adapters). +Please refer to that repo for a guide on how to contribute to `dbt-spark`. -## About this document -This document is a guide intended for folks interested in contributing to `dbt-spark`. Below, we document the process by which members of the community should create issues and submit pull requests (PRs) in this repository. It is not intended as a guide for using `dbt-spark`, and it assumes a certain level of familiarity with Python concepts such as virtualenvs, `pip`, Python modules, and so on. This guide assumes you are using macOS or Linux and are comfortable with the command line. - -For those wishing to contribute we highly suggest reading the dbt-core's [contribution guide](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md) if you haven't already. Almost all of the information there is applicable to contributing here, too! - -### Signing the CLA - -Please note that all contributors to `dbt-spark` must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements) to have their Pull Request merged into an `dbt-spark` codebase. If you are unable to sign the CLA, then the `dbt-spark` maintainers will unfortunately be unable to merge your Pull Request. You are, however, welcome to open issues and comment on existing ones. - - -## Getting the code - -You will need `git` in order to download and modify the `dbt-spark` source code. You can find directions [here](https://github.com/git-guides/install-git) on how to install `git`. - -### External contributors - -If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt-spark` by forking the `dbt-spark` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to: - -1. fork the `dbt-spark` repository -2. clone your fork locally -3. check out a new branch for your proposed changes -4. push changes to your fork -5. open a pull request against `dbt-labs/dbt-spark` from your forked repository - -### dbt Labs contributors - -If you are a member of the `dbt Labs` GitHub organization, you will have push access to the `dbt-spark` repo. Rather than forking `dbt-spark` to make your changes, just clone the repository, check out a new branch, and push directly to that branch. - - -## Running `dbt-spark` in development - -### Installation - -First make sure that you set up your `virtualenv` as described in [Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment). Ensure you have the latest version of pip installed with `pip install --upgrade pip`. Next, install `dbt-spark` latest dependencies: - -```sh -pip install -e . -r dev-requirements.txt -``` - -When `dbt-spark` is installed this way, any changes you make to the `dbt-spark` source code will be reflected immediately in your next `dbt-spark` run. - -To confirm you have correct version of `dbt-core` installed please run `dbt --version` and `which dbt`. - - -## Testing - -### Initial Setup - -`dbt-spark` uses test credentials specified in a `test.env` file in the root of the repository. This `test.env` file is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing. To create your `test.env` file, copy the provided example file, then supply your relevant credentials. - -``` -cp test.env.example test.env -$EDITOR test.env -``` - -### Test commands -There are a few methods for running tests locally. - -#### dagger -To run functional tests we rely on [dagger](https://dagger.io/). This launches a virtual container or containers to test against. - -```sh -pip install -r dagger/requirements.txt -python dagger/run_dbt_spark_tests.py --profile databricks_sql_endpoint --test-path tests/functional/adapter/test_basic.py::TestSimpleMaterializationsSpark::test_base -``` - -`--profile`: required, this is the kind of spark connection to test against - -_options_: - - "apache_spark" - - "spark_session" - - "spark_http_odbc" - - "databricks_sql_endpoint" - - "databricks_cluster" - - "databricks_http_cluster" - -`--test-path`: optional, this is the path to the test file you want to run. If not specified, all tests will be run. - -#### pytest -Finally, you can also run a specific test or group of tests using `pytest` directly (if you have all the dependencies set up on your machine). With a Python virtualenv active and dev dependencies installed you can do things like: - -```sh -# run all functional tests -python -m pytest --profile databricks_sql_endpoint tests/functional/ -# run specific functional tests -python -m pytest --profile databricks_sql_endpoint tests/functional/adapter/test_basic.py -# run all unit tests in a file -python -m pytest tests/unit/test_adapter.py -# run a specific unit test -python -m pytest test/unit/test_adapter.py::TestSparkAdapter::test_profile_with_database -``` -## Updating Docs - -Many changes will require and update to the `dbt-spark` docs here are some useful resources. - -- Docs are [here](https://docs.getdbt.com/). -- The docs repo for making changes is located [here]( https://github.com/dbt-labs/docs.getdbt.com). -- The changes made are likely to impact one or both of [Spark Profile](https://docs.getdbt.com/reference/warehouse-profiles/spark-profile), or [Saprk Configs](https://docs.getdbt.com/reference/resource-configs/spark-configs). -- We ask every community member who makes a user-facing change to open an issue or PR regarding doc changes. - -## Adding CHANGELOG Entry - -We use [changie](https://changie.dev) to generate `CHANGELOG` entries. **Note:** Do not edit the `CHANGELOG.md` directly. Your modifications will be lost. - -Follow the steps to [install `changie`](https://changie.dev/guide/installation/) for your system. - -Once changie is installed and your PR is created, simply run `changie new` and changie will walk you through the process of creating a changelog entry. Commit the file that's created and your changelog entry is complete! - -You don't need to worry about which `dbt-spark` version your change will go into. Just create the changelog entry with `changie`, and open your PR against the `main` branch. All merged changes will be included in the next minor version of `dbt-spark`. The Core maintainers _may_ choose to "backport" specific changes in order to patch older minor versions. In that case, a maintainer will take care of that backport after merging your PR, before releasing the new version of `dbt-spark`. - -## Submitting a Pull Request - -dbt Labs provides a CI environment to test changes to the `dbt-spark` adapter, and periodic checks against the development version of `dbt-core` through Github Actions. - -A `dbt-spark` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or functional test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code. - -Once all requests and answers have been answered the `dbt-spark` maintainer can trigger CI testing. - -Once all tests are passing and your PR has been approved, a `dbt-spark` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada: +If you have already opened a pull request and need to migrate it to the new repo, please refer to the +[contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#submitting-a-pull-request). diff --git a/README.md b/README.md index adeb16915..526ece339 100644 --- a/README.md +++ b/README.md @@ -7,89 +7,5 @@ </a> </p> -**[dbt](https://www.getdbt.com/)** enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications. - -dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis. - -## dbt-spark - -The `dbt-spark` package contains all of the code enabling dbt to work with Apache Spark and Databricks. For -more information, consult [the docs](https://docs.getdbt.com/docs/profile-spark). - -## Getting started - -- [Install dbt](https://docs.getdbt.com/docs/installation) -- Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) - -## Running locally -A `docker-compose` environment starts a Spark Thrift server and a Postgres database as a Hive Metastore backend. -Note: dbt-spark now supports Spark 3.3.2. - -The following command starts two docker containers: - -```sh -docker-compose up -d -``` - -It will take a bit of time for the instance to start, you can check the logs of the two containers. -If the instance doesn't start correctly, try the complete reset command listed below and then try start again. - -Create a profile like this one: - -```yaml -spark_testing: - target: local - outputs: - local: - type: spark - method: thrift - host: 127.0.0.1 - port: 10000 - user: dbt - schema: analytics - connect_retries: 5 - connect_timeout: 60 - retry_all: true -``` - -Connecting to the local spark instance: - -* The Spark UI should be available at [http://localhost:4040/sqlserver/](http://localhost:4040/sqlserver/) -* The endpoint for SQL-based testing is at `http://localhost:10000` and can be referenced with the Hive or Spark JDBC drivers using connection string `jdbc:hive2://localhost:10000` and default credentials `dbt`:`dbt` - -Note that the Hive metastore data is persisted under `./.hive-metastore/`, and the Spark-produced data under `./.spark-warehouse/`. To completely reset you environment run the following: - -```sh -docker-compose down -rm -rf ./.hive-metastore/ -rm -rf ./.spark-warehouse/ -``` - -#### Additional Configuration for MacOS - -If installing on MacOS, use `homebrew` to install required dependencies. - ```sh - brew install unixodbc - ``` - -### Reporting bugs and contributing code - -- Want to report a bug or request a feature? Let us know on [Slack](http://slack.getdbt.com/), or open [an issue](https://github.com/fishtown-analytics/dbt-spark/issues/new). - -## Code of Conduct - -Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [PyPA Code of Conduct](https://www.pypa.io/en/latest/code-of-conduct/). - -## Join the dbt Community - -- Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) -- Read more on the [dbt Community Discourse](https://discourse.getdbt.com) - -## Reporting bugs and contributing code - -- Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt-spark/issues/new) -- Want to help us build dbt? Check out the [Contributing Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md) - -## Code of Conduct - -Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct). +This repository as moved into the `dbt-labs/dbt-adapters` monorepo found +[here](https://www.github.com/dbt-labs/dbt-adapters). diff --git a/test.env.example b/test.env.example deleted file mode 100644 index 140da4042..000000000 --- a/test.env.example +++ /dev/null @@ -1,11 +0,0 @@ -# databricks credentials -DBT_DATABRICKS_HOST_NAME=<{this value}.cloud.databricks.com>.cloud.databricks.com -DBT_DATABRICKS_TOKEN=<personal token> -DBT_DATABRICKS_CLUSTER_NAME=<sql/protocolv1/o/{not this}/{this value}> -ODBC_DRIVER=</file/path/to/local/ODBC.driver> -DBT_DATABRICKS_ENDPOINT=</sql/1.0/warehouses/{this value}> - -# users for testing 'grants' functionality -DBT_TEST_USER_1= -DBT_TEST_USER_2= -DBT_TEST_USER_3=