Feat: use PostgreSQL for the database backend (#131)

Author: lalver1

.devcontainer/devcontainer.json

@@ -2,9 +2,10 @@
   "name": "caltrans/pems",
   "dockerComposeFile": ["../compose.yml"],
   "service": "dev",
+  "runServices": ["dev", "pgweb"],
   "forwardPorts": ["docs:8000"],
   "workspaceFolder": "/caltrans/app",
-  "postStartCommand": ["/bin/bash", "bin/reset_db.sh"],
+  "postStartCommand": ["/bin/bash", "bin/setup.sh"],
   "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"],
   "customizations": {
     "vscode": {

.env.sample

@@ -3,12 +3,23 @@ DJANGO_SUPERUSER_USERNAME=pems-admin
 DJANGO_SUPERUSER_EMAIL=pems-admin@compiler.la
 DJANGO_SUPERUSER_PASSWORD=superuser12345!
 
-# Django storage
+# Django Database settings
 DJANGO_DB_RESET=true
-DJANGO_STORAGE_DIR=.
-DJANGO_DB_FILE=django.db
+DJANGO_DB_NAME=django
+DJANGO_DB_USER=django
+DJANGO_DB_PASSWORD=django_password
 DJANGO_DB_FIXTURES="pems/local_fixtures.json"
 
+# PostgreSQL settings
+POSTGRES_HOSTNAME=postgres
+POSTGRES_DB=postgres
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres_password
+POSTGRES_PORT=5432
+POSTGRES_SSLMODE=disable
+
+PGWEB_PORT=8081
+
 # Streamlit
 STREAMLIT_LOCAL_PORT=8501
 # options: hidden, sidebar

.github/workflows/tests-pytest.yml

@@ -17,11 +17,6 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Install system packages
-        run: |
-          sudo apt-get update -y
-          sudo apt-get install -y gettext
-
       - uses: actions/setup-python@v5
         with:
           python-version-file: .github/workflows/.python-version
@@ -33,9 +28,6 @@ jobs:
           pip install -e .[test]
           pip install -r streamlit_app/requirements.txt
 
-      - name: Run setup
-        run: ./bin/init.sh
-
       - name: Run tests
         run: ./tests/pytest/run.sh
 

appcontainer/Dockerfile

@@ -91,6 +91,9 @@ ENV GUNICORN_CONF "/$USER/run/gunicorn.conf.py"
 # overwrite default nginx.conf
 COPY appcontainer/nginx.conf /etc/nginx/nginx.conf
 
+# copy certs for PostgreSQL verify-full
+COPY appcontainer/certs/aws_global_postgres_ca_bundle.pem app/certs/aws_global_postgres_ca_bundle.pem
+
 WORKDIR /$USER/app
 
 # copy runtime files

appcontainer/certs/aws_global_postgres_ca_bundle.pem

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -ex
+
+# Ensure databases, users, migrations, and superuser are set up
+should_reset=${REMOTE_CONTAINERS:-false}
+if [[ $should_reset == "true" ]]; then
+    # running in a devcontainer, reset the DB
+    python manage.py ensure_db --reset
+else
+    python manage.py ensure_db
+fi
+
+# Load data fixtures (if any)
+valid_fixtures=$(echo "$DJANGO_DB_FIXTURES" | grep -e fixtures\.json$ || test $? = 1)
+
+if [[ -n "$valid_fixtures" ]]; then
+    python manage.py loaddata $DJANGO_DB_FIXTURES
+else
+    echo "No JSON fixtures to load"
+fi

bin/init.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 set -eu
 
-# initialize Django
+# prepare static files
 
-bin/init.sh
+python manage.py collectstatic --no-input
 
 # start the web server
 

bin/reset_db.sh

@@ -1,30 +1,14 @@
 #!/usr/bin/env bash
 set -eu
 
-#
 # S3 bucket name is injected by Copilot as an environment variable
 # since it was created via copilot storage init --name pems-db, the variable is 'PEMSDB_NAME'
 S3_BUCKET_NAME="$PEMSDB_NAME"
 S3_FIXTURE_PATH="fixtures.json"
-LOCAL_FIXTURE_PATH="fixtures.json"
 
 echo "Downloading $S3_FIXTURE_PATH from bucket $S3_BUCKET_NAME"
-aws s3 cp "s3://${S3_BUCKET_NAME}/${S3_FIXTURE_PATH}" "${LOCAL_FIXTURE_PATH}"
+aws s3 cp "s3://${S3_BUCKET_NAME}/${S3_FIXTURE_PATH}" "${DJANGO_DB_FIXTURES}"
 echo "Download complete"
 
-# initialize Django
-
-bin/init.sh
-
-# effectively reset database by loading downloaded fixtures into the database
-echo "Loading data from ${LOCAL_FIXTURE_PATH}"
-python manage.py loaddata "${LOCAL_FIXTURE_PATH}"
-echo "Data loading complete"
-
-# start the web server
-
-nginx
-
-# start the application server
-
-python -m gunicorn -c $GUNICORN_CONF pems.wsgi
+bin/setup.sh
+bin/start.sh