Feat: use PostgreSQL for the database backend

.devcontainer/devcontainer.json

@@ -2,9 +2,10 @@
   "name": "caltrans/pems",
   "dockerComposeFile": ["../compose.yml"],
   "service": "dev",
+  "runServices": ["dev", "pgweb"],
   "forwardPorts": ["docs:8000"],
   "workspaceFolder": "/caltrans/app",
-  "postStartCommand": ["/bin/bash", "bin/reset_db.sh"],
+  "postStartCommand": ["/bin/bash", "bin/setup.sh"],
   "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"],
   "customizations": {
     "vscode": {

.env.sample

@@ -3,12 +3,23 @@ DJANGO_SUPERUSER_USERNAME=pems-admin
 DJANGO_SUPERUSER_EMAIL=pems-admin@compiler.la
 DJANGO_SUPERUSER_PASSWORD=superuser12345!
 
-# Django storage
+# Django Database settings
 DJANGO_DB_RESET=true
-DJANGO_STORAGE_DIR=.
-DJANGO_DB_FILE=django.db
+DJANGO_DB_NAME=django
+DJANGO_DB_USER=django
+DJANGO_DB_PASSWORD=django_password
 DJANGO_DB_FIXTURES="pems/local_fixtures.json"
 
+# PostgreSQL settings
+POSTGRES_HOSTNAME=postgres
+POSTGRES_DB=postgres
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres_password
+POSTGRES_PORT=5432
+POSTGRES_SSLMODE=disable
+
+PGWEB_PORT=8081
+
 # Streamlit
 STREAMLIT_LOCAL_PORT=8501
 # options: hidden, sidebar

.github/workflows/tests-pytest.yml

@@ -17,11 +17,6 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Install system packages
-        run: |
-          sudo apt-get update -y
-          sudo apt-get install -y gettext
-
       - uses: actions/setup-python@v5
         with:
           python-version-file: .github/workflows/.python-version
@@ -33,8 +28,8 @@ jobs:
           pip install -e .[test]
           pip install -r streamlit_app/requirements.txt
 
-      - name: Run setup
-        run: ./bin/init.sh
+      - name: Collect static files
+        run: python manage.py collectstatic --no-input
 
       - name: Run tests
         run: ./tests/pytest/run.sh

appcontainer/Dockerfile

@@ -91,6 +91,9 @@ ENV GUNICORN_CONF "/$USER/run/gunicorn.conf.py"
 # overwrite default nginx.conf
 COPY appcontainer/nginx.conf /etc/nginx/nginx.conf
 
+# copy certs for PostgreSQL verify-full
+COPY appcontainer/certs/aws_global_postgres_ca_bundle.pem app/certs/aws_global_postgres_ca_bundle.pem
+
 WORKDIR /$USER/app
 
 # copy runtime files

bin/setup.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -ex
+
+# Ensure databases, users, migrations, and superuser are set up
+should_reset=${REMOTE_CONTAINERS:-false}
+if [[ $should_reset == "true" ]]; then
+    # running in a devcontainer, reset the DB
+    python manage.py ensure_db --reset
+else
+    python manage.py ensure_db
+fi
+
+# Load data fixtures (if any)
+valid_fixtures=$(echo "$DJANGO_DB_FIXTURES" | grep -e fixtures\.json$ || test $? = 1)
+
+if [[ -n "$valid_fixtures" ]]; then
+    python manage.py loaddata $DJANGO_DB_FIXTURES
+else
+    echo "No JSON fixtures to load"
+fi

bin/start.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 set -eu
 
-# initialize Django
+# prepare static files
 
-bin/init.sh
+python manage.py collectstatic --no-input
 
 # start the web server
 

bin/start_aws.sh

@@ -1,30 +1,29 @@
 #!/usr/bin/env bash
 set -eu
 
-#
 # S3 bucket name is injected by Copilot as an environment variable
 # since it was created via copilot storage init --name pems-db, the variable is 'PEMSDB_NAME'
 S3_BUCKET_NAME="$PEMSDB_NAME"
 S3_FIXTURE_PATH="fixtures.json"
-LOCAL_FIXTURE_PATH="fixtures.json"
+DJANGO_DB_FIXTURES="fixtures.json"
 
 echo "Downloading $S3_FIXTURE_PATH from bucket $S3_BUCKET_NAME"
-aws s3 cp "s3://${S3_BUCKET_NAME}/${S3_FIXTURE_PATH}" "${LOCAL_FIXTURE_PATH}"
+aws s3 cp "s3://${S3_BUCKET_NAME}/${S3_FIXTURE_PATH}" "${DJANGO_DB_FIXTURES}"
 echo "Download complete"
 
-# initialize Django
+# PostgreSQL database settings (username, host, dbname, password, port) are injected by Copilot as an environment variable
+# called 'POSTGRESWEB_SECRET' since the database was created via
+# copilot storage init -l workload -t Aurora -w web -n postgres-web --engine PostgreSQL --initial-db django
 
-bin/init.sh
+python manage.py migrate
 
-# effectively reset database by loading downloaded fixtures into the database
-echo "Loading data from ${LOCAL_FIXTURE_PATH}"
-python manage.py loaddata "${LOCAL_FIXTURE_PATH}"
-echo "Data loading complete"
+# Load data fixtures (if any)
+valid_fixtures=$(echo "$DJANGO_DB_FIXTURES" | grep -e fixtures\.json$ || test $? = 1)
 
-# start the web server
+if [[ -n "$valid_fixtures" ]]; then
+    python manage.py loaddata $DJANGO_DB_FIXTURES
+else
+    echo "No JSON fixtures to load"
+fi
 
-nginx
-
-# start the application server
-
-python -m gunicorn -c $GUNICORN_CONF pems.wsgi
+bin/start.sh

compose.yml

@@ -6,6 +6,10 @@ services:
       context: .
       dockerfile: appcontainer/Dockerfile
     image: caltrans/pems:web
+    command: ["-c", "bin/setup.sh && exec bin/start.sh"]
+    depends_on:
+      postgres:
+        condition: service_healthy
     env_file: .env
     ports:
       - "${DJANGO_LOCAL_PORT:-8000}:8000"
@@ -15,13 +19,44 @@ services:
       context: .
       dockerfile: .devcontainer/Dockerfile
     image: caltrans/pems:dev
+    depends_on:
+      - postgres
+      - pgweb
     env_file: .env
     # https://code.visualstudio.com/docs/remote/create-dev-container#_use-docker-compose
     entrypoint: sleep infinity
     volumes:
       - ./:/caltrans/app
       - ${HOME}/.aws:/home/caltrans/.aws
 
+  postgres:
+    image: postgres:16
+    environment:
+      - POSTGRES_DB
+      - POSTGRES_USER
+      - POSTGRES_PASSWORD
+    healthcheck:
+      test: ["CMD", "pg_isready", "-d", "${POSTGRES_DB}", "-U", "${POSTGRES_USER}"]
+      interval: 10s
+      timeout: 60s
+      retries: 6
+      start_period: 10s
+    ports:
+      - "${POSTGRES_PORT:-5432}:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+
+  pgweb:
+    container_name: pgweb
+    image: sosedoff/pgweb
+    ports:
+      - "${PGWEB_PORT:-8081}:8081"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      - PGWEB_DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOSTNAME}:${POSTGRES_PORT}/${POSTGRES_DB}?sslmode=disable
+
   docs:
     image: caltrans/pems:dev
     entrypoint: mkdocs
@@ -41,3 +76,7 @@ services:
       - "${STREAMLIT_LOCAL_PORT:-8501}:8501"
     volumes:
       - ./:/caltrans/app
+
+volumes:
+  pgdata:
+    driver: local

infra/copilot/web/addons/postgres-web.yml

@@ -0,0 +1,133 @@
+Parameters:
+  App:
+    Type: String
+    Description: Your application's name.
+  Env:
+    Type: String
+    Description: The environment name your service, job, or workflow is being deployed to.
+  Name:
+    Type: String
+    Description: Your workload's name.
+  # Customize your Aurora Serverless cluster by setting the default value of the following parameters.
+  postgreswebDBName:
+    Type: String
+    Description: The name of the initial database to be created in the Aurora Serverless v2 cluster.
+    Default: django
+    # Cannot have special characters
+    # Naming constraints: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.Constraints
+Mappings:
+  postgreswebEnvScalingConfigurationMap:
+    dev:
+      "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128
+      "DBMaxCapacity": 8   # AllowedValues: from 0.5 through 128
+
+    All:
+      "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128
+      "DBMaxCapacity": 8   # AllowedValues: from 0.5 through 128
+
+Resources:
+  postgreswebDBSubnetGroup:
+    Type: 'AWS::RDS::DBSubnetGroup'
+    Properties:
+      DBSubnetGroupDescription: Group of Copilot private subnets for Aurora Serverless v2 cluster.
+      SubnetIds:
+        !Split [',', { 'Fn::ImportValue': !Sub '${App}-${Env}-PrivateSubnets' }]
+  postgreswebSecurityGroup:
+    Metadata:
+      'aws:copilot:description': 'A security group for your workload to access the Aurora Serverless v2 cluster postgresweb'
+    Type: 'AWS::EC2::SecurityGroup'
+    Properties:
+      GroupDescription: !Sub 'The Security Group for ${Name} to access Aurora Serverless v2 cluster postgresweb.'
+      VpcId:
+        Fn::ImportValue:
+          !Sub '${App}-${Env}-VpcId'
+      Tags:
+        - Key: Name
+          Value: !Sub 'copilot-${App}-${Env}-${Name}-Aurora'
+  postgreswebDBClusterSecurityGroup:
+    Metadata:
+      'aws:copilot:description': 'A security group for your Aurora Serverless v2 cluster postgresweb'
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: The Security Group for the Aurora Serverless v2 cluster.
+      SecurityGroupIngress:
+        - ToPort: 5432
+          FromPort: 5432
+          IpProtocol: tcp
+          Description: !Sub 'From the Aurora Security Group of the workload ${Name}.'
+          SourceSecurityGroupId: !Ref postgreswebSecurityGroup
+      VpcId:
+        Fn::ImportValue:
+          !Sub '${App}-${Env}-VpcId'
+      Tags:
+        - Key: Name
+          Value: !Sub 'copilot-${App}-${Env}-${Name}-Aurora'
+  postgreswebAuroraSecret:
+    Metadata:
+      'aws:copilot:description': 'A Secrets Manager secret to store your DB credentials'
+    Type: AWS::SecretsManager::Secret
+    Properties:
+      Description: !Sub Aurora main user secret for ${AWS::StackName}
+      GenerateSecretString:
+        SecretStringTemplate: '{"username": "postgres"}'
+        GenerateStringKey: "password"
+        ExcludePunctuation: true
+        IncludeSpace: false
+        PasswordLength: 16
+  postgreswebDBClusterParameterGroup:
+    Metadata:
+      'aws:copilot:description': 'A DB parameter group for engine configuration values'
+    Type: 'AWS::RDS::DBClusterParameterGroup'
+    Properties:
+      Description: !Ref 'AWS::StackName'
+      Family: 'aurora-postgresql16'
+      Parameters:
+        client_encoding: 'UTF8'
+  postgreswebDBCluster:
+    Metadata:
+      'aws:copilot:description': 'The postgresweb Aurora Serverless v2 database cluster'
+    Type: 'AWS::RDS::DBCluster'
+    Properties:
+      MasterUsername:
+        !Join [ "",  [ '{{resolve:secretsmanager:', !Ref postgreswebAuroraSecret, ":SecretString:username}}" ]]
+      MasterUserPassword:
+        !Join [ "",  [ '{{resolve:secretsmanager:', !Ref postgreswebAuroraSecret, ":SecretString:password}}" ]]
+      DatabaseName: !Ref postgreswebDBName
+      Engine: 'aurora-postgresql'
+      EngineVersion: '16.2'
+      DBClusterParameterGroupName: !Ref postgreswebDBClusterParameterGroup
+      DBSubnetGroupName: !Ref postgreswebDBSubnetGroup
+      Port: 5432
+      VpcSecurityGroupIds:
+        - !Ref postgreswebDBClusterSecurityGroup
+      ServerlessV2ScalingConfiguration:
+        # Replace "All" below with "!Ref Env" to set different autoscaling limits per environment.
+        MinCapacity: !FindInMap [postgreswebEnvScalingConfigurationMap, All, DBMinCapacity]
+        MaxCapacity: !FindInMap [postgreswebEnvScalingConfigurationMap, All, DBMaxCapacity]
+  postgreswebDBWriterInstance:
+    Metadata:
+      'aws:copilot:description': 'The postgresweb Aurora Serverless v2 writer instance'
+    Type: 'AWS::RDS::DBInstance'
+    Properties:
+      DBClusterIdentifier: !Ref postgreswebDBCluster
+      DBInstanceClass: db.serverless
+      Engine: 'aurora-postgresql'
+      PromotionTier: 1
+      AvailabilityZone: !Select
+        - 0
+        - !GetAZs
+          Ref: AWS::Region
+
+  postgreswebSecretAuroraClusterAttachment:
+    Type: AWS::SecretsManager::SecretTargetAttachment
+    Properties:
+      SecretId: !Ref postgreswebAuroraSecret
+      TargetId: !Ref postgreswebDBCluster
+      TargetType: AWS::RDS::DBCluster
+Outputs:
+  postgreswebSecret: # injected as POSTGRESWEB_SECRET environment variable by Copilot.
+    Description: "The JSON secret that holds the database username and password. Fields are 'host', 'port', 'dbname', 'username', 'password', 'dbClusterIdentifier' and 'engine'"
+    Value: !Ref postgreswebAuroraSecret
+  postgreswebSecurityGroup:
+    Description: "The security group to attach to the workload."
+    Value: !Ref postgreswebSecurityGroup