Refactor: pems_streamlit package (#184)

Author: thekaveman

.devcontainer/Dockerfile

@@ -50,10 +50,9 @@ ENV PATH="$PATH:/$USER/.local/bin" \
     PYTHONUSERBASE="/$USER/.local"
 
 # install devcontainer requirements
-RUN pip install -e .[dev,test]
-
+RUN pip install .[dev,test]
 # install docs requirements
 RUN pip install --no-cache-dir -r docs/requirements.txt
 
-# install streamlit requirements
-RUN pip install --no-cache-dir -r streamlit_app/requirements.txt
+# install source packages as editable
+RUN pip install -e ./pems_streamlit -e ./pems_web

.github/workflows/tests-pytest.yml

@@ -25,8 +25,7 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          pip install -e .[test]
-          pip install -r streamlit_app/requirements.txt
+          pip install .[test] ./pems_streamlit ./pems_web
 
       - name: Run tests
         run: ./tests/pytest/run.sh

.vscode/launch.json

@@ -33,7 +33,7 @@
       "type": "debugpy",
       "request": "launch",
       "module": "streamlit",
-      "args": ["run", "streamlit_app/main.py"],
+      "args": ["run", "pems_streamlit/src/pems_streamlit/main.py"],
       "env": {
         "PYTHONBUFFERED": "1",
         "PYTHONWARNINGS": "default",
@@ -45,7 +45,7 @@
       "type": "debugpy",
       "request": "launch",
       "module": "streamlit",
-      "args": ["run", "streamlit_app/main.py"],
+      "args": ["run", "pems_streamlit/src/pems_streamlit/main.py"],
       "env": {
         "PYTHONBUFFERED": "1",
         "PYTHONWARNINGS": "default",

compose.yml

@@ -69,13 +69,11 @@ services:
   streamlit:
     build:
       context: .
-      dockerfile: streamlit_app/Dockerfile
+      dockerfile: pems_streamlit/container/Dockerfile
     image: caltrans/pems:streamlit
     env_file: .env
     ports:
       - "${STREAMLIT_LOCAL_PORT:-8501}:8501"
-    volumes:
-      - ./:/caltrans/app
 
 volumes:
   pgdata:

infra/copilot/streamlit/manifest.yml

@@ -17,7 +17,7 @@ http:
 image:
   # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/backend-service/#image-build
   build:
-    dockerfile: ../streamlit_app/Dockerfile
+    dockerfile: ../pems_streamlit/container/Dockerfile
     context: ../
   # Port exposed through your container to route traffic to it.
   port: 8501

pems_streamlit/container/Dockerfile

@@ -0,0 +1,61 @@
+ARG PYTHON_VERSION=3.12
+
+# multi-stage build
+#
+# stage 1: builds the package wheel from source
+#          using the git metadata for version info
+FROM python:${PYTHON_VERSION} AS build_wheel
+WORKDIR /build
+
+# upgrade pip, install build dependencies
+RUN python -m pip install --upgrade pip build setuptools_scm
+
+# copy source files
+COPY . .
+RUN git config --global --add safe.directory /build
+
+# Move into directory to run the build
+WORKDIR /build/pems_streamlit
+
+# build package
+RUN python -m build
+
+# multi-stage build
+#
+# stage 2: installs the wheel in a fresh base container
+#          using the pre-built package, and copying only needed source
+FROM python:${PYTHON_VERSION} AS app_container
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    USER=caltrans
+
+EXPOSE 8501
+
+# create non-root $USER and home directory
+RUN useradd --create-home --shell /bin/bash $USER && \
+    python -m pip install --upgrade pip
+
+COPY LICENSE LICENSE
+
+# switch to non-root $USER
+USER $USER
+
+# update env for local pip installs
+# see https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUSERBASE
+# since all `pip install` commands are in the context of $USER
+# $PYTHONUSERBASE is the location used by default
+ENV PATH="$PATH:/$USER/.local/bin" \
+    PYTHONUSERBASE="/$USER/.local" \
+    PYTHONPATH="$PYTHONPATH:/$USER/app"
+
+WORKDIR /$USER/app
+
+COPY .streamlit .streamlit
+COPY pems_streamlit/container/entrypoint.sh entrypoint.sh
+COPY pems_streamlit/container/run.py run.py
+COPY --from=build_wheel /build/pems_streamlit/dist /wheels
+
+RUN pip install $(find /wheels -name pems_streamlit*.whl)
+
+ENTRYPOINT ["./entrypoint.sh"]

streamlit_app/entrypoint.sh renamed to pems_streamlit/container/entrypoint.sh

@@ -7,7 +7,8 @@ BASE_URL_PATH=${STREAMLIT_BASE_URL:-""}
 echo "Starting Streamlit with base URL path: ${BASE_URL_PATH}"
 
 # Execute the Streamlit command, passing the base URL path
-exec streamlit run streamlit_app/main.py \
+# Use the run.py helper since streamlit run wants the path to a python file
+exec streamlit run run.py \
   --server.port=8501 \
   --server.address=0.0.0.0 \
   --server.baseUrlPath="${BASE_URL_PATH}"

pems_streamlit/container/run.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+import runpy
+
+# since we're installing pems_streamlit as a package now, the main.py file won't exist
+# in the running container, so we can't point `streamlit run` at it
+
+# instead, use runpy to run the pems_streamlit.main module directly.
+
+if __name__ == "__main__":
+    runpy.run_module("pems_streamlit.main", run_name="__main__", alter_sys=True)

pems_streamlit/pyproject.toml

@@ -0,0 +1,27 @@
+[project]
+name = "pems_streamlit"
+description = "The Streamlit application for PeMS data visualizations."
+dynamic = ["version"]
+requires-python = ">=3.12"
+dependencies = [
+    "boto3==1.39.7",
+    "django==5.2.3",
+    "pandas==2.3.0",
+    "streamlit==1.45.1",
+]
+
+[build-system]
+requires = ["setuptools>=75", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["pems_streamlit*"]
+namespaces = false
+
+[tool.setuptools.package-data]
+pems_streamlit = ["*.txt"]
+
+[tool.setuptools_scm]
+# Tell scm to look one directory up for the .git folder
+root = ".."