feat: parallelized build steps and uploading/downloading artifacts

.github/workflows/publish.yaml

@@ -1,44 +1,136 @@
 name: Build, Test, and Publish
 
-# triggers: whenever there is new changes pulled/pushed on this 
+# triggers: whenever there is new changes pulled/pushed on this
 # repo under given conditions, run the below jobs
 on:
   pull_request:
     types: [opened, synchronize]
   push:
     branches:
       - main
+  # Manually trigger a workflow
   # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch
   workflow_dispatch:
 
 jobs:
 
-  build-test-and-publish:
+  check-verison-txt:
     runs-on: ubuntu-latest
     steps:
-    # github actions checksout, clones our repo, and checks out the branch we're working in
-    - uses: actions/checkout@v3
+      # github actions checksout, clones our repo, and checks out the branch we're working in
+      - uses: actions/checkout@v4
+        with:
+          # Number of commits to fetch. 0 indicates all history for all branches and tags
+          # fetching all tags so to aviod duplicate version tagging in 'Tag with the Release Version'
+          fetch-depth: 0
+      # tagging the release version to avoid duplicate releases
+      - name: Tag with the Release Version
+        run: |
+          git tag $(cat version.txt)
+
+
+  lint-format-and-static-code-checks:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+    # caching dependencies
+    - uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install pre-commit
+      run: |
+        pip install pre-commit
+    - name: Lint, Format, and Other Static Code Quality Check
+      run: |
+        /bin/bash -x run.sh lint:ci
+
+
+  build-wheel-and-sdist:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
     - name: Set up Python 3.8
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
-    - name: Run
+    - uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install build CLI
       run: |
-        /bin/bash run.sh install
-        /bin/bash run.sh build
-        /bin/bash run.sh publish:test
+        pip install build
+    - name: Build Python Package
+      run: |
+        /bin/bash -x run.sh build
+
+    # uploading the built package to publish in the publish workflow
+    - name: Upload wheel and sdist
+      uses: actions/upload-artifact@v4
+      with:
+        name: wheel-and-sdist-artifact
+        path: ./dist/*
 
+
+  publish:
+    needs:
+      - check-verison-txt
+      - lint-format-and-static-code-checks
+      - build-wheel-and-sdist
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.8
+
+    # downloading the built package in 'build-wheel-and-sdist' workflow to publish
+    - name: Download wheel and sdist
+      uses: actions/download-artifact@v4
+      with:
+        name: wheel-and-sdist-artifact
+        path: ./dist/
+
+    - uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install twine CLI
+      run: |
+        pip install twine
+
+    - name: Publish to Test PyPI
+      # setting -x in below publish:test will not leak any secrets as they are masked in github
+      run: |
+        /bin/bash -x run.sh publish:test
       env:
         TEST_PYPI_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }}
+
+    - name: Publish to Prod PyPI
+      run: |
+        /bin/bash -x run.sh publish:prod
+      env:
         PROD_PYPI_TOKEN: ${{ secrets.PROD_PYPI_TOKEN }}
-
-    # - name: Install dependencies
-    #   run: |
-    #     python -m pip install --upgrade pip
-    #     pip install pre-commit
-    # - name: Running pre-commit hooks
-    #   run: |
-    #     SKIP=no-commit-to-branch pre-commit run --all-files
+
+    - name: Push Tags
+      run: |
+       git push origin --tags
+
+
+
 
 
   # https://docs.github.com/en/actions/learn-github-actions/contexts#example-printing-context-information-to-the-log
@@ -74,4 +166,4 @@ jobs:
           SECRETS_CONTEXT: ${{ toJson(secrets) }}
         run: echo "$SECRETS_CONTEXT"
       - name: Dump Variables
-        run: echo "${{ toJson(vars) }}"
+        run: echo "${{ toJson(vars) }}"

README.md

@@ -774,4 +774,139 @@ twine upload --help
 
 ## Detailed CI/CD Workflow for Python Packages
 
-![Detailed CI/CD Workflow for Python Packages](https://github.com/avr2002/python-packaging/blob/main/packaging_demo/assets/detailed-workflow.png?raw=true)
+![Detailed CI/CD Workflow for Python Packages](https://github.com/avr2002/python-packaging/blob/main/packaging_demo/assets/detailed-workflow.png?raw=true)
+
+
+### GitHub CI/CD Workflow in worflows yaml file
+
+```yaml
+# .github/workflows/publish.yaml
+
+name: Build, Test, and Publish
+
+# triggers: whenever there is new changes pulled/pushed on this
+# repo under given conditions, run the below jobs
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+  push:
+    branches:
+      - main
+
+  # Manually trigger a workflow
+  # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch
+  workflow_dispatch:
+
+jobs:
+
+  build-test-and-publish:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    # github actions checksout, clones our repo, and checks out the branch we're working in
+    - uses: actions/checkout@v3
+      with:
+        # Number of commits to fetch. 0 indicates all history for all branches and tags
+        # fetching all tags so to aviod duplicate version tagging in 'Tag with the Release Version'
+        fetch-depth: 0
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.8
+
+    # tagging the release version to avoid duplicate releases
+    - name: Tag with the Release Version
+      run: |
+        git tag $(cat version.txt)
+
+    - name: Install Python Dependencies
+      run: |
+        /bin/bash -x run.sh install
+
+    - name: Lint, Format, and Other Static Code Quality Check
+      run: |
+        /bin/bash -x run.sh lint:ci
+
+    - name: Build Python Package
+      run: |
+        /bin/bash -x run.sh build
+
+    - name: Publish to Test PyPI
+      # setting -x in below publish:test will not leak any secrets as they are masked in github
+      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      run: |
+        /bin/bash -x run.sh publish:test
+      env:
+        TEST_PYPI_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }}
+
+    - name: Publish to Prod PyPI
+      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      run: |
+        /bin/bash -x run.sh publish:prod
+      env:
+        PROD_PYPI_TOKEN: ${{ secrets.PROD_PYPI_TOKEN }}
+
+    - name: Push Tags
+      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      run: |
+       git push origin --tags
+```
+
+### GitHub Actions Optimizations
+
+1. Locking Requirements
+   - It's not really recommended to pin exact versions of dependencies to avoid future conflict
+   - But it's good practice to store them in the requirements file for future debugging.
+   - Tools:
+
+2. Dependency Caching
+   - Whenever github actions gets executed in the github CI, everytime it's run on a fresh container.
+    Thus, everytime we'll have to download and re-install dependencies from pip again and again;
+    which is not a good as it's inefficeint and slows our workflow.
+
+   - Thus we would want to install all the dependencies when the workflow ran first and use it every
+     time a new worflow is run.
+
+   - GitHub Actions provide this functionality by caching the dependencies, it stores the installed
+     dependencies(`~/.cache/pip`) and downloads it everytime a new workflow is run.
+     [**Docs**](https://github.com/actions/cache/blob/main/examples.md#python---pip)
+
+   ```toml
+   - uses: actions/cache@v3
+     with:
+      path: ~/.cache/pip
+      key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+      restore-keys: |
+        ${{ runner.os }}-pip-
+   ```
+
+3. Parallelization
+
+   - We moved from above shown workflow to now a parallelized workflow as shown below.
+   - This helps in faster running of workflow, helping discover bugs in any steps
+     at the same time which was not possible in linear flow as earlier.
+
+```yaml
+# See .github/workflows/publish.yaml
+
+jobs:
+
+  check-verison-txt:
+    ...
+
+  lint-format-and-static-code-checks:
+    ....
+
+  build-wheel-and-sdist:
+    ...
+
+  publish:
+    needs:
+      - check-verison-txt
+      - lint-format-and-static-code-checks
+      - build-wheel-and-sdist
+    ...
+```

pyproject.toml

@@ -12,8 +12,8 @@ description = "Demo for Python Packaging"
 readme = "README.md"
 requires-python = ">=3.8"
 keywords = [
-    "python", "bash", "makefile", "pypi", "ci-cd", "setuptools", "wheels", 
-    "package-development", "github-actions", "pypi-package", "pre-commit-hooks", 
+    "python", "bash", "makefile", "pypi", "ci-cd", "setuptools", "wheels",
+    "package-development", "github-actions", "pypi-package", "pre-commit-hooks",
     "pyproject-toml", "gitactions-workflow", "github-actions-enabled", "pre-commit-ci",
     "pre-commit-config"
 ]

run.sh

@@ -11,7 +11,7 @@ function load-dotenv {
     # adding this if condition so that this does not fail in
     # github actions CI
     if [ ! -f "$THIS_DIR/.env" ]; then
-        echo "no .env file found"
+        echo "No .env file found"
         return 1
     fi
 
@@ -25,10 +25,14 @@ function install {
     python -m pip install --editable "${THIS_DIR}/[dev]"
 }
 
-function lint {
+function lint:ci {
     SKIP=no-commit-to-branch pre-commit run --all-files
 }
 
+function lint {
+    pre-commit run --all-files
+}
+
 function build {
     python -m build --sdist --wheel "${THIS_DIR}"
 }

version.txt

@@ -1 +1 @@
-v0.0.6
+v0.0.8