From 97168a551592028efa18a45473dd9e2118528571 Mon Sep 17 00:00:00 2001 From: Jacky Hu Date: Thu, 31 Oct 2024 19:26:05 -0700 Subject: [PATCH 1/4] Fix CI after the connector package split --- .github/workflows/code-quality-checks.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 80ac94a7c..30d7780ad 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -2,16 +2,17 @@ name: Code Quality Checks on: push: branches: - - main + - PECO-1803/connector-split pull_request: branches: - - main + - PECO-1803/connector-split jobs: run-unit-tests: runs-on: ubuntu-latest strategy: matrix: python-version: [3.8, 3.9, "3.10", "3.11"] + package: ["databricks_sql_connector_core", "databricks_sql_connector"] steps: #---------------------------------------------- # check-out repo and set-up python @@ -48,21 +49,26 @@ jobs: - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root + working-directory: ${{ github.workspace }}/${{ matrix.package }} #---------------------------------------------- # install your root project, if required #---------------------------------------------- - name: Install library run: poetry install --no-interaction + working-directory: ${{ github.workspace }}/${{ matrix.package }} #---------------------------------------------- # run test suite #---------------------------------------------- - name: Run tests + if: maxtrix.package != "databricks_sql_connector" run: poetry run python -m pytest tests/unit + working-directory: ${{ github.workspace }}/${{ matrix.package }} check-linting: runs-on: ubuntu-latest strategy: matrix: python-version: [3.8, 3.9, "3.10"] + package: ["databricks_sql_connector_core"] steps: #---------------------------------------------- # check-out repo and set-up python @@ -99,22 +105,26 @@ jobs: - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root + working-directory: ${{ github.workspace }}/${{ matrix.package }} #---------------------------------------------- # install your root project, if required #---------------------------------------------- - name: Install library run: poetry install --no-interaction + working-directory: ${{ github.workspace }}/${{ matrix.package }} #---------------------------------------------- # black the code #---------------------------------------------- - name: Black run: poetry run black --check src + working-directory: ${{ github.workspace }}/${{ matrix.package }} check-types: runs-on: ubuntu-latest strategy: matrix: python-version: [3.8, 3.9, "3.10"] + package: ["databricks_sql_connector_core"] steps: #---------------------------------------------- # check-out repo and set-up python @@ -151,11 +161,13 @@ jobs: - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root + working-directory: ${{ github.workspace }}/${{ matrix.package }} #---------------------------------------------- # install your root project, if required #---------------------------------------------- - name: Install library run: poetry install --no-interaction + working-directory: ${{ github.workspace }}/${{ matrix.package }} #---------------------------------------------- # mypy the code #---------------------------------------------- @@ -163,3 +175,5 @@ jobs: run: | mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 poetry run mypy --install-types --non-interactive src + working-directory: ${{ github.workspace }}/${{ matrix.package }} + From bb4e5dc344d285464beb9f50fda7511aad4bf883 Mon Sep 17 00:00:00 2001 From: Jacky Hu Date: Thu, 31 Oct 2024 19:36:26 -0700 Subject: [PATCH 2/4] Fix syntax --- .github/workflows/code-quality-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 30d7780ad..add4479ac 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -60,7 +60,7 @@ jobs: # run test suite #---------------------------------------------- - name: Run tests - if: maxtrix.package != "databricks_sql_connector" + if: ${{ maxtrix.package != 'databricks_sql_connector'}} run: poetry run python -m pytest tests/unit working-directory: ${{ github.workspace }}/${{ matrix.package }} check-linting: From 915e5504bb3efa0528cef8b31f89993772e598e0 Mon Sep 17 00:00:00 2001 From: Jacky Hu Date: Thu, 31 Oct 2024 20:11:55 -0700 Subject: [PATCH 3/4] Refactor action to add resuable build workflow --- .github/workflows/build.yml | 76 ++++++++++ .github/workflows/code-quality-checks.yml | 168 +--------------------- 2 files changed, 82 insertions(+), 162 deletions(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..1b4f6a1c3 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,76 @@ +name: Build and validate package + +on: + workflow_call: + inputs: + target_path: + description: "The path of the package or service to build" + required: true + type: string + run_test: + description: "Run lint and test" + required: true + type: boolean + +jobs: + build_package: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ 3.8, 3.9, "3.10", "3.11" ] + steps: + - name: Set up python ${{ matrix.python-version }} + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # load cached venv if cache exists + #---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v2 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + working-directory: ${{ github.workspace }}/${{ github.event.inputs.target_path }} + + #---------------------------------------------- + # install your root project, if required + #---------------------------------------------- + - name: Install library + run: poetry install --no-interaction + working-directory: ${{ github.event.inputs.target_path }} + + - name: Check-linting + if: ${{ inputs.run_test }} + run: poetry run black --check src + working-directory: ${{ github.event.inputs.target_path }} + + - name: Run unit test + if: ${{ inputs.run_test }} + run: poetry run python -m pytest tests/unit + working-directory: ${{ github.event.inputs.target_path }} + +# - name: Check types +# if: ${{ inputs.run_test }} +# run: | +# mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 +# poetry run mypy --install-types --non-interactive src +# working-directory: ${{ github.event.inputs.target_path }} \ No newline at end of file diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index add4479ac..8d05bb766 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -7,173 +7,17 @@ on: branches: - PECO-1803/connector-split jobs: - run-unit-tests: + validate-connector-core: runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8, 3.9, "3.10", "3.11"] - package: ["databricks_sql_connector_core", "databricks_sql_connector"] steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - name: Check out repository uses: actions/checkout@v2 - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v2 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - working-directory: ${{ github.workspace }}/${{ matrix.package }} - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: poetry install --no-interaction - working-directory: ${{ github.workspace }}/${{ matrix.package }} - #---------------------------------------------- - # run test suite - #---------------------------------------------- - - name: Run tests - if: ${{ maxtrix.package != 'databricks_sql_connector'}} - run: poetry run python -m pytest tests/unit - working-directory: ${{ github.workspace }}/${{ matrix.package }} - check-linting: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8, 3.9, "3.10"] - package: ["databricks_sql_connector_core"] - steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Check out repository - uses: actions/checkout@v2 - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v2 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - working-directory: ${{ github.workspace }}/${{ matrix.package }} - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: poetry install --no-interaction - working-directory: ${{ github.workspace }}/${{ matrix.package }} - #---------------------------------------------- - # black the code - #---------------------------------------------- - - name: Black - run: poetry run black --check src - working-directory: ${{ github.workspace }}/${{ matrix.package }} - - check-types: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8, 3.9, "3.10"] - package: ["databricks_sql_connector_core"] - steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Check out repository - uses: actions/checkout@v2 - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true + ref: ${{ github.head_ref }} - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v2 + - name: Build + uses: ./.github/workflows/build.yml with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - working-directory: ${{ github.workspace }}/${{ matrix.package }} - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: poetry install --no-interaction - working-directory: ${{ github.workspace }}/${{ matrix.package }} - #---------------------------------------------- - # mypy the code - #---------------------------------------------- - - name: Mypy - run: | - mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 - poetry run mypy --install-types --non-interactive src - working-directory: ${{ github.workspace }}/${{ matrix.package }} + target_path: "databricks_sql_connector_core" + run_test: true From 21c982f06d6c6f0b1d35ee9edb776bb78e423d6e Mon Sep 17 00:00:00 2001 From: Jacky Hu Date: Thu, 31 Oct 2024 20:34:43 -0700 Subject: [PATCH 4/4] Change checkout to v3 --- .github/workflows/code-quality-checks.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 8d05bb766..6ebb9ebd5 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -11,9 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v2 - with: - ref: ${{ github.head_ref }} + uses: actions/checkout@v3 - name: Build uses: ./.github/workflows/build.yml