Merge remote-tracking branch 'origin/master' into prod

Author: Lee-W

.flake8

@@ -7,7 +7,8 @@ assignees: ""
 ---
 
 ## Type
-* [ ] Conent inaccurate
+
+* [ ] Content inaccurate
 * [ ] Content missing
 * [ ] Typo
 

.github/ISSUE_TEMPLATE/documentation.md

@@ -13,9 +13,10 @@
 ## Description
 <!--Describe what the change is**-->
 
-## Checklist:
+## Checklist
+
 - [ ] Add test cases to all the changes you introduce
-- [ ] Run `poetry run pytest` locally to ensure all linter checks pass
+- [ ] Run `make lint` and `make test` locally to ensure all linter checks and testing pass
 - [ ] Update the documentation if necessary
 
 ## Steps to Test This Pull Request

.github/pull_request_template.md

@@ -12,17 +12,20 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v2
+      - name: Check out
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v1
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: 3.8
+          enable-cache: true
+          version: "latest"
 
       - name: Install dependencies
         run: |
-          pip install -r requirements.txt -r requirements-dev.txt -c constraints-3.8.txt
-          pip install black==19.10b0 click==7.1.2
+          uv sync --group dev
 
       - name: Run linters
         run: make lint

.github/workflows/python.yml

@@ -1,7 +1,31 @@
 ARG AIRFLOW_VERSION=1.10.15
 ARG PYTHON_VERSION=3.8
+ARG PLATFORM=linux/amd64
 
-FROM apache/airflow:${AIRFLOW_VERSION}-python${PYTHON_VERSION}
+FROM --platform=${PLATFORM} ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-bookworm-slim AS builder
+
+ENV UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0
+
+RUN apt-get update && \
+    apt-get install -y gcc libc6-dev --no-install-recommends
+
+WORKDIR /app
+
+COPY ./pyproject.toml .
+COPY ./uv.lock .
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --frozen --no-install-project --no-dev
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-install-project --no-dev
+
+
+FROM --platform=${PLATFORM} apache/airflow:${AIRFLOW_VERSION}-python${PYTHON_VERSION}
 
 USER root
 
@@ -21,13 +45,10 @@ RUN chmod +x /entrypoint.sh
 
 USER airflow
 
-COPY ./requirements.txt ${AIRFLOW_HOME}/requirements.txt
-COPY ./constraints-3.8.txt ${AIRFLOW_HOME}/constraints-3.8.txt
-
-RUN pip install --no-cache-dir -r ${AIRFLOW_HOME}/requirements.txt --constraint constraints-3.8.txt
+COPY --from=builder --chown=airflow:airflow /app /app
+ENV PATH="/app/.venv/bin:$PATH"
 
 COPY airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
-
 COPY --chown=airflow:root dags ${AIRFLOW_HOME}/dags
 
 ENTRYPOINT ["/entrypoint.sh"]

Dockerfile

@@ -1,7 +1,30 @@
 ARG AIRFLOW_VERSION=1.10.15
 ARG PYTHON_VERSION=3.8
+ARG PLATFORM=linux/amd64
 
-FROM apache/airflow:${AIRFLOW_VERSION}-python${PYTHON_VERSION}
+FROM --platform=${PLATFORM} ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-bookworm-slim AS builder
+
+ENV UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0
+
+RUN apt-get update && \
+    apt-get install -y gcc libc6-dev --no-install-recommends
+
+WORKDIR /app
+
+COPY ./pyproject.toml .
+COPY ./uv.lock .
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --frozen --no-install-project --group dev
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --group dev
+
+FROM --platform=${PLATFORM} apache/airflow:${AIRFLOW_VERSION}-python${PYTHON_VERSION}
 
 USER root
 
@@ -21,14 +44,10 @@ RUN chmod +x /entrypoint.sh
 
 USER airflow
 
-COPY ./requirements.txt ${AIRFLOW_HOME}/requirements.txt
-COPY ./requirements-dev.txt ${AIRFLOW_HOME}/requirements-dev.txt
-COPY ./constraints-3.8.txt ${AIRFLOW_HOME}/constraints-3.8.txt
-
-RUN pip install --no-cache-dir -r ${AIRFLOW_HOME}/requirements.txt -r ${AIRFLOW_HOME}/requirements-dev.txt --constraint constraints-3.8.txt
+COPY --from=builder --chown=airflow:airflow /app /app
+ENV PATH="/app/.venv/bin:$PATH"
 
 COPY airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
-
 COPY --chown=airflow:root dags ${AIRFLOW_HOME}/dags
 
 ENV AIRFLOW_TEST_MODE=True

Dockerfile.test

@@ -1,18 +1,17 @@
 lint:
-	black . --check
-	isort --check-only .
-	flake8 .
-	mypy dags/ tests/
+	uv run ruff check .
+	uv run ruff format .
+	uv run mypy dags/ tests/
 
 format:
-	black .
-	isort .
+	uv run ruff check . --fix
+	uv run ruff format .
 
 test:
-	PYTHONPATH=./dags pytest
+	PYTHONPATH=./dags uv run pytest
 
 coverage:
-	PYTHONPATH=./dags pytest --cov=dags tests
+	PYTHONPATH=./dags uv run pytest --cov=dags tests
 
 build-dev:
 	docker-compose -f ./docker-compose-dev.yml build

Makefile

@@ -21,45 +21,33 @@ Using Airflow to implement our ETL pipelines.
 - [Python 3.8+](https://www.python.org/downloads/release/python-3811/)
 - [Docker](https://docs.docker.com/get-docker/)
 - [Git](https://git-scm.com/book/zh-tw/v2/%E9%96%8B%E5%A7%8B-Git-%E5%AE%89%E8%A3%9D%E6%95%99%E5%AD%B8)
-- [venv](https://docs.python.org/3/library/venv.html)
+- [uv]
 
 ## Installation
 
-There are several tools available to create a virtual environment in Python.
+We use [uv] to manage dependencies and virtual environment.
 
-Below are the steps to manage a virtual environment using `venv`:
+Below are the steps to create a virtual environment using [uv]:
 
-1. Create a Virtual Environment
+1. Create a Virtual Environment with Dependencies Installed
 
     To create a virtual environment, run the following command:
 
     ```bash
-    python -m venv venv
+    uv sync
     ```
 
-    In this example, `venv` is the name of the virtual environment directory, but you can replace it with any name you prefer.
+    By default, [uv] sets up the virtual environment in `.venv`
 
 2. Activate the Virtual Environment
 
     After creating the virtual environment, activate it using the following command:
 
     ```bash
-    source venv/bin/activate
+    source .venv/bin/activate
     ```
 
-3. Install Dependencies
-
-    After activating the virtual environment, you can install the required dependencies:
-
-    ```bash
-    # Install airflow and dev dependencies
-    pip install -r requirements.txt -r requirements-dev.txt -c constraints-3.8.txt
-
-    # black is conflict with click, so install it separately
-    pip install black==19.10b0 click==7.1.2
-    ```
-
-4. Deactivate the Virtual Environment
+3. Deactivate the Virtual Environment
 
     When you're done working in the virtual environment, you can deactivate it with:
 
@@ -82,8 +70,9 @@ Below are the steps to manage a virtual environment using `venv`:
 ### BigQuery (Optional)
 
 Set up the Authentication for GCP: <https://googleapis.dev/python/google-api-core/latest/auth.html>
-    *After running `gcloud auth application-default login`, you will get a credentials.json file located at `$HOME/.config/gcloud/application_default_credentials.json`. Run `export GOOGLE_APPLICATION_CREDENTIALS="/path/to/keyfile.json"` if you have it.
-    * service-account.json: Please contact @david30907d via email or Discord. You do not need this json file if you are running the sandbox staging instance for development.
+
+- After running `gcloud auth application-default login`, you will get a credentials.json file located at `$HOME/.config/gcloud/application_default_credentials.json`. Run `export GOOGLE_APPLICATION_CREDENTIALS="/path/to/keyfile.json"` if you have it.
+- service-account.json: Please contact @david30907d via email or Discord. You do not need this json file if you are running the sandbox staging instance for development.
 
 ## Running the Project
 
@@ -108,7 +97,7 @@ make down-dev
 
 > The difference between production and dev/test compose files is that the dev/test compose file uses a locally built image, while the production compose file uses the image from Docker Hub.
 
-If you are a authorized maintainer, you can pull the image from the GCP Artifact Registry.
+If you are an authorized maintainer, you can pull the image from the GCP Artifact Registry.
 
 Docker client must be configured to use the GCP Artifact Registry.
 
@@ -136,3 +125,5 @@ Please check the [Production Deployment Guide](./docs/DEPLOYMENT.md).
 ## Contact
 
 [PyCon TW Volunteer Data Team - Discord](https://discord.com/channels/752904426057892052/900721883383758879)
+
+[uv]: https://docs.astral.sh/uv/