diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 1efbf0a..2361de0 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -1,6 +1,6 @@ name: Build, Test, and Publish -# triggers: whenever there is new changes pulled/pushed on this +# triggers: whenever there is new changes pulled/pushed on this # repo under given conditions, run the below jobs on: pull_request: @@ -8,37 +8,129 @@ on: push: branches: - main + # Manually trigger a workflow # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch workflow_dispatch: jobs: - build-test-and-publish: + check-verison-txt: runs-on: ubuntu-latest steps: - # github actions checksout, clones our repo, and checks out the branch we're working in - - uses: actions/checkout@v3 + # github actions checksout, clones our repo, and checks out the branch we're working in + - uses: actions/checkout@v4 + with: + # Number of commits to fetch. 0 indicates all history for all branches and tags + # fetching all tags so to aviod duplicate version tagging in 'Tag with the Release Version' + fetch-depth: 0 + # tagging the release version to avoid duplicate releases + - name: Tag with the Release Version + run: | + git tag $(cat version.txt) + + + lint-format-and-static-code-checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.8 + uses: actions/setup-python@v4 + with: + python-version: 3.8 + # caching dependencies + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install pre-commit + run: | + pip install pre-commit + - name: Lint, Format, and Other Static Code Quality Check + run: | + /bin/bash -x run.sh lint:ci + + + build-wheel-and-sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 - name: Set up Python 3.8 - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: 3.8 - - name: Run + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install build CLI run: | - /bin/bash run.sh install - /bin/bash run.sh build - /bin/bash run.sh publish:test + pip install build + - name: Build Python Package + run: | + /bin/bash -x run.sh build + + # uploading the built package to publish in the publish workflow + - name: Upload wheel and sdist + uses: actions/upload-artifact@v4 + with: + name: wheel-and-sdist-artifact + path: ./dist/* + + publish: + needs: + - check-verison-txt + - lint-format-and-static-code-checks + - build-wheel-and-sdist + runs-on: ubuntu-latest + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.8 + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + # downloading the built package in 'build-wheel-and-sdist' workflow to publish + - name: Download wheel and sdist + uses: actions/download-artifact@v4 + with: + name: wheel-and-sdist-artifact + path: ./dist/ + + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install twine CLI + run: | + pip install twine + + - name: Publish to Test PyPI + # setting -x in below publish:test will not leak any secrets as they are masked in github + run: | + /bin/bash -x run.sh publish:test env: TEST_PYPI_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }} + + - name: Publish to Prod PyPI + run: | + /bin/bash -x run.sh publish:prod + env: PROD_PYPI_TOKEN: ${{ secrets.PROD_PYPI_TOKEN }} - - # - name: Install dependencies - # run: | - # python -m pip install --upgrade pip - # pip install pre-commit - # - name: Running pre-commit hooks - # run: | - # SKIP=no-commit-to-branch pre-commit run --all-files + + - name: Push Tags + run: | + git push origin --tags + + + # https://docs.github.com/en/actions/learn-github-actions/contexts#example-printing-context-information-to-the-log @@ -74,4 +166,4 @@ jobs: SECRETS_CONTEXT: ${{ toJson(secrets) }} run: echo "$SECRETS_CONTEXT" - name: Dump Variables - run: echo "${{ toJson(vars) }}" \ No newline at end of file + run: echo "${{ toJson(vars) }}" diff --git a/README.md b/README.md index 11a07b4..e9b11f3 100644 --- a/README.md +++ b/README.md @@ -774,4 +774,139 @@ twine upload --help ## Detailed CI/CD Workflow for Python Packages -![Detailed CI/CD Workflow for Python Packages](https://github.com/avr2002/python-packaging/blob/main/packaging_demo/assets/detailed-workflow.png?raw=true) \ No newline at end of file +![Detailed CI/CD Workflow for Python Packages](https://github.com/avr2002/python-packaging/blob/main/packaging_demo/assets/detailed-workflow.png?raw=true) + + +### GitHub CI/CD Workflow in worflows yaml file + +```yaml +# .github/workflows/publish.yaml + +name: Build, Test, and Publish + +# triggers: whenever there is new changes pulled/pushed on this +# repo under given conditions, run the below jobs +on: + pull_request: + types: [opened, synchronize] + + push: + branches: + - main + + # Manually trigger a workflow + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch + workflow_dispatch: + +jobs: + + build-test-and-publish: + + runs-on: ubuntu-latest + + steps: + # github actions checksout, clones our repo, and checks out the branch we're working in + - uses: actions/checkout@v3 + with: + # Number of commits to fetch. 0 indicates all history for all branches and tags + # fetching all tags so to aviod duplicate version tagging in 'Tag with the Release Version' + fetch-depth: 0 + + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: 3.8 + + # tagging the release version to avoid duplicate releases + - name: Tag with the Release Version + run: | + git tag $(cat version.txt) + + - name: Install Python Dependencies + run: | + /bin/bash -x run.sh install + + - name: Lint, Format, and Other Static Code Quality Check + run: | + /bin/bash -x run.sh lint:ci + + - name: Build Python Package + run: | + /bin/bash -x run.sh build + + - name: Publish to Test PyPI + # setting -x in below publish:test will not leak any secrets as they are masked in github + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + run: | + /bin/bash -x run.sh publish:test + env: + TEST_PYPI_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }} + + - name: Publish to Prod PyPI + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + run: | + /bin/bash -x run.sh publish:prod + env: + PROD_PYPI_TOKEN: ${{ secrets.PROD_PYPI_TOKEN }} + + - name: Push Tags + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + run: | + git push origin --tags +``` + +### GitHub Actions Optimizations + +1. Locking Requirements + - It's not really recommended to pin exact versions of dependencies to avoid future conflict + - But it's good practice to store them in the requirements file for future debugging. + - Tools: + +2. Dependency Caching + - Whenever github actions gets executed in the github CI, everytime it's run on a fresh container. + Thus, everytime we'll have to download and re-install dependencies from pip again and again; + which is not a good as it's inefficeint and slows our workflow. + + - Thus we would want to install all the dependencies when the workflow ran first and use it every + time a new worflow is run. + + - GitHub Actions provide this functionality by caching the dependencies, it stores the installed + dependencies(`~/.cache/pip`) and downloads it everytime a new workflow is run. + [**Docs**](https://github.com/actions/cache/blob/main/examples.md#python---pip) + + ```toml + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ``` + +3. Parallelization + + - We moved from above shown workflow to now a parallelized workflow as shown below. + - This helps in faster running of workflow, helping discover bugs in any steps + at the same time which was not possible in linear flow as earlier. + +```yaml +# See .github/workflows/publish.yaml + +jobs: + + check-verison-txt: + ... + + lint-format-and-static-code-checks: + .... + + build-wheel-and-sdist: + ... + + publish: + needs: + - check-verison-txt + - lint-format-and-static-code-checks + - build-wheel-and-sdist + ... +``` diff --git a/pyproject.toml b/pyproject.toml index 67c85be..c846373 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,8 +12,8 @@ description = "Demo for Python Packaging" readme = "README.md" requires-python = ">=3.8" keywords = [ - "python", "bash", "makefile", "pypi", "ci-cd", "setuptools", "wheels", - "package-development", "github-actions", "pypi-package", "pre-commit-hooks", + "python", "bash", "makefile", "pypi", "ci-cd", "setuptools", "wheels", + "package-development", "github-actions", "pypi-package", "pre-commit-hooks", "pyproject-toml", "gitactions-workflow", "github-actions-enabled", "pre-commit-ci", "pre-commit-config" ] diff --git a/run.sh b/run.sh index 0ef6e8e..e071cbe 100755 --- a/run.sh +++ b/run.sh @@ -11,7 +11,7 @@ function load-dotenv { # adding this if condition so that this does not fail in # github actions CI if [ ! -f "$THIS_DIR/.env" ]; then - echo "no .env file found" + echo "No .env file found" return 1 fi @@ -25,10 +25,14 @@ function install { python -m pip install --editable "${THIS_DIR}/[dev]" } -function lint { +function lint:ci { SKIP=no-commit-to-branch pre-commit run --all-files } +function lint { + pre-commit run --all-files +} + function build { python -m build --sdist --wheel "${THIS_DIR}" } diff --git a/version.txt b/version.txt index 254a9f2..3ce186f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.6 \ No newline at end of file +v0.0.8