Feat: add test and prod deployment environments (#149)

Author: lalver1

.github/workflows/deploy.yml

@@ -15,6 +15,12 @@ defaults:
   run:
     shell: bash
 
+concurrency:
+  # this ternary operator-like expression gives us the name of the deployment environment (see https://docs.github.com/en/actions/learn-github-actions/expressions#example)
+  # and ensures that only one deployment per environment is in progress at a time
+  group: ${{ github.ref_type != 'tag' && 'dev' || contains(github.ref, '-rc') && 'test' || 'prod' }}
+  cancel-in-progress: true
+
 jobs:
   tests-ui:
     uses: ./.github/workflows/tests-ui.yml
@@ -28,6 +34,7 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     needs: [tests-ui, tests-pytest]
+    environment: ${{ github.ref_type != 'tag' && 'dev' || contains(github.ref, '-rc') && 'test' || 'prod' }}
     permissions:
       packages: write
       id-token: write
@@ -80,13 +87,11 @@ jobs:
           sudo mv ./.tools/copilot /usr/local/bin/copilot
 
       - name: Deploy web Service
-        run: |
-          copilot deploy --name web --env dev
+        run: copilot deploy --name web --env ${{ github.environment }}
         working-directory: ./infra
 
       - name: Deploy streamlit Service
-        run: |
-          copilot deploy --name streamlit --env dev
+        run: copilot deploy --name streamlit --env ${{ github.environment }}
         working-directory: ./infra
 
   release:

bin/start_aws.sh

@@ -1,13 +1,12 @@
 #!/usr/bin/env bash
 set -eu
 
-# S3 bucket name is injected by Copilot as an environment variable
-# since it was created via copilot storage init --name pems-db, the variable is 'PEMSDB_NAME'
-S3_BUCKET_NAME="$PEMSDB_NAME"
+# S3 bucket name (BUCKET_NAME) is injected by Copilot as an environment variable
+# in the service manifest
 S3_FIXTURE_PATH="fixtures.json"
 
-echo "Downloading $S3_FIXTURE_PATH from bucket $S3_BUCKET_NAME"
-aws s3 cp "s3://${S3_BUCKET_NAME}/${S3_FIXTURE_PATH}" "${DJANGO_DB_FIXTURES}"
+echo "Downloading $S3_FIXTURE_PATH from bucket $BUCKET_NAME"
+aws s3 cp "s3://${BUCKET_NAME}/${S3_FIXTURE_PATH}" "${DJANGO_DB_FIXTURES}"
 echo "Download complete"
 
 bin/setup.sh

infra/cloudformation/gh_actions.yml

@@ -58,7 +58,10 @@ Resources:
               - Sid: AssumeRole
                 Effect: Allow
                 Action: "sts:AssumeRole"
-                Resource: "arn:aws:iam::715841364638:role/pems-dev-EnvManagerRole"
+                Resource:
+                  - "arn:aws:iam::715841364638:role/pems-dev-EnvManagerRole"
+                  - "arn:aws:iam::715841364638:role/pems-test-EnvManagerRole"
+                  - "arn:aws:iam::715841364638:role/pems-prod-EnvManagerRole"
               - Sid: ListStackInstances
                 Effect: Allow
                 Action: "cloudformation:ListStackInstances"
@@ -67,6 +70,10 @@ Resources:
                 Effect: Allow
                 Action: "cloudformation:DescribeStacks"
                 Resource: "arn:aws:cloudformation:us-west-2:715841364638:stack/StackSet-pems-infrastructure-c95edbbc-22a7-4239-a10b-7d73ba9344c4/45d26e70-31d1-11f0-a04a-0a0876def005"
+              - Sid: UpdateStack
+                Effect: Allow
+                Action: "cloudformation:UpdateStack"
+                Resource: "arn:aws:cloudformation:us-west-2:715841364638:stack/pems-github-actions/407a9060-575b-11f0-84e3-0a0aa26e756d"
               - Sid: GetAuthorizationToken
                 Effect: Allow
                 Action: "ecr:GetAuthorizationToken"

infra/copilot/environments/addons/addons.parameters.yml

@@ -0,0 +1,3 @@
+Parameters:
+  VPCID: !Ref VPC
+  PrivateSubnets: !Join [",", [!Ref PrivateSubnet1, !Ref PrivateSubnet2]]

infra/copilot/web/addons/postgres-web.yml renamed to infra/copilot/environments/addons/postgres.yml

@@ -4,63 +4,81 @@ Parameters:
     Description: Your application's name.
   Env:
     Type: String
-    Description: The environment name your service, job, or workflow is being deployed to.
-  Name:
-    Type: String
-    Description: Your workload's name.
+    Description: The name of the environment being deployed.
   # Customize your Aurora Serverless cluster by setting the default value of the following parameters.
-  postgreswebDBName:
+  postgresDBName:
     Type: String
     Description: The name of the initial database to be created in the Aurora Serverless v2 cluster.
     Default: postgres
     # Cannot have special characters
     # Naming constraints: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.Constraints
+  VPCID:
+    Type: String
+    Description: The ID of the VPC in which to create the Aurora Serverless v2 cluster.
+    Default: ""
+  PrivateSubnets:
+    Type: String
+    Description: The IDs of the private subnets in which to create the Aurora Serverless v2 cluster.
+    Default: ""
+
 Mappings:
-  postgreswebEnvScalingConfigurationMap:
+  postgresEnvScalingConfigurationMap:
     dev:
       "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128
       "DBMaxCapacity": 8 # AllowedValues: from 0.5 through 128
 
+    prod:
+      "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128
+      "DBMaxCapacity": 8 # AllowedValues: from 0.5 through 128
+
+    test:
+      "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128
+      "DBMaxCapacity": 8 # AllowedValues: from 0.5 through 128
+
     All:
       "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128
       "DBMaxCapacity": 8 # AllowedValues: from 0.5 through 128
 
 Resources:
-  postgreswebDBSubnetGroup:
+  postgresDBSubnetGroup:
     Type: "AWS::RDS::DBSubnetGroup"
     Properties:
-      DBSubnetGroupDescription: Group of Copilot private subnets for Aurora Serverless v2 cluster.
-      SubnetIds:
-        !Split [",", { "Fn::ImportValue": !Sub "${App}-${Env}-PrivateSubnets" }]
-  postgreswebSecurityGroup:
+      DBSubnetGroupDescription: Group of private subnets for Aurora Serverless v2 cluster.
+      SubnetIds: !Split [",", !Ref PrivateSubnets]
+
+  postgresWorkloadSecurityGroup:
     Metadata:
-      "aws:copilot:description": "A security group for your workload to access the Aurora Serverless v2 cluster postgresweb"
+      "aws:copilot:description": "A security group for one or more workloads to access the Aurora Serverless v2 cluster postgres"
     Type: "AWS::EC2::SecurityGroup"
     Properties:
-      GroupDescription: !Sub "The Security Group for ${Name} to access Aurora Serverless v2 cluster postgresweb."
-      VpcId:
-        Fn::ImportValue: !Sub "${App}-${Env}-VpcId"
+      GroupDescription: "The Security Group to access Aurora Serverless v2 cluster postgres."
+      VpcId: !Ref VPCID
       Tags:
         - Key: Name
-          Value: !Sub "copilot-${App}-${Env}-${Name}-Aurora"
-  postgreswebDBClusterSecurityGroup:
+          Value: !Sub "copilot-${App}-${Env}-Aurora"
+
+  postgresDBClusterSecurityGroup:
     Metadata:
-      "aws:copilot:description": "A security group for your Aurora Serverless v2 cluster postgresweb"
+      "aws:copilot:description": "A security group for your Aurora Serverless v2 cluster postgres"
     Type: AWS::EC2::SecurityGroup
     Properties:
       GroupDescription: The Security Group for the Aurora Serverless v2 cluster.
-      SecurityGroupIngress:
-        - ToPort: 5432
-          FromPort: 5432
-          IpProtocol: tcp
-          Description: !Sub "From the Aurora Security Group of the workload ${Name}."
-          SourceSecurityGroupId: !Ref postgreswebSecurityGroup
-      VpcId:
-        Fn::ImportValue: !Sub "${App}-${Env}-VpcId"
+      VpcId: !Ref VPCID
       Tags:
         - Key: Name
-          Value: !Sub "copilot-${App}-${Env}-${Name}-Aurora"
-  postgreswebAuroraSecret:
+          Value: !Sub "copilot-${App}-${Env}-Aurora"
+
+  postgresDBClusterSecurityGroupIngressFromWorkload:
+    Type: AWS::EC2::SecurityGroupIngress
+    Properties:
+      Description: Ingress from one or more workloads in the environment.
+      GroupId: !Ref postgresDBClusterSecurityGroup
+      IpProtocol: tcp
+      ToPort: 5432
+      FromPort: 5432
+      SourceSecurityGroupId: !Ref postgresWorkloadSecurityGroup
+
+  postgresAuroraSecret:
     Metadata:
       "aws:copilot:description": "A Secrets Manager secret to store your DB credentials"
     Type: AWS::SecretsManager::Secret
@@ -72,7 +90,7 @@ Resources:
         ExcludePunctuation: true
         IncludeSpace: false
         PasswordLength: 16
-  postgreswebDBClusterParameterGroup:
+  postgresDBClusterParameterGroup:
     Metadata:
       "aws:copilot:description": "A DB parameter group for engine configuration values"
     Type: "AWS::RDS::DBClusterParameterGroup"
@@ -81,17 +99,18 @@ Resources:
       Family: "aurora-postgresql16"
       Parameters:
         client_encoding: "UTF8"
-  postgreswebDBCluster:
+
+  postgresDBCluster:
     Metadata:
-      "aws:copilot:description": "The postgresweb Aurora Serverless v2 database cluster"
+      "aws:copilot:description": "The postgres Aurora Serverless v2 database cluster"
     Type: "AWS::RDS::DBCluster"
     Properties:
       MasterUsername:
         !Join [
           "",
           [
             "{{resolve:secretsmanager:",
-            !Ref postgreswebAuroraSecret,
+            !Ref postgresAuroraSecret,
             ":SecretString:username}}",
           ],
         ]
@@ -100,31 +119,31 @@ Resources:
           "",
           [
             "{{resolve:secretsmanager:",
-            !Ref postgreswebAuroraSecret,
+            !Ref postgresAuroraSecret,
             ":SecretString:password}}",
           ],
         ]
-      DatabaseName: !Ref postgreswebDBName
+      DatabaseName: !Ref postgresDBName
       Engine: "aurora-postgresql"
       EngineVersion: "16.2"
-      EnableHttpEndpoint: true # enable the Data API feature
-      DBClusterParameterGroupName: !Ref postgreswebDBClusterParameterGroup
-      DBSubnetGroupName: !Ref postgreswebDBSubnetGroup
+      DBClusterParameterGroupName: !Ref postgresDBClusterParameterGroup
+      DBSubnetGroupName: !Ref postgresDBSubnetGroup
       Port: 5432
       VpcSecurityGroupIds:
-        - !Ref postgreswebDBClusterSecurityGroup
+        - !Ref postgresDBClusterSecurityGroup
       ServerlessV2ScalingConfiguration:
         # Replace "All" below with "!Ref Env" to set different autoscaling limits per environment.
         MinCapacity:
-          !FindInMap [postgreswebEnvScalingConfigurationMap, All, DBMinCapacity]
+          !FindInMap [postgresEnvScalingConfigurationMap, All, DBMinCapacity]
         MaxCapacity:
-          !FindInMap [postgreswebEnvScalingConfigurationMap, All, DBMaxCapacity]
-  postgreswebDBWriterInstance:
+          !FindInMap [postgresEnvScalingConfigurationMap, All, DBMaxCapacity]
+
+  postgresDBWriterInstance:
     Metadata:
-      "aws:copilot:description": "The postgresweb Aurora Serverless v2 writer instance"
+      "aws:copilot:description": "The postgres Aurora Serverless v2 writer instance"
     Type: "AWS::RDS::DBInstance"
     Properties:
-      DBClusterIdentifier: !Ref postgreswebDBCluster
+      DBClusterIdentifier: !Ref postgresDBCluster
       DBInstanceClass: db.serverless
       Engine: "aurora-postgresql"
       PromotionTier: 1
@@ -133,16 +152,21 @@ Resources:
         - !GetAZs
           Ref: AWS::Region
 
-  postgreswebSecretAuroraClusterAttachment:
+  postgresSecretAuroraClusterAttachment:
     Type: AWS::SecretsManager::SecretTargetAttachment
     Properties:
-      SecretId: !Ref postgreswebAuroraSecret
-      TargetId: !Ref postgreswebDBCluster
+      SecretId: !Ref postgresAuroraSecret
+      TargetId: !Ref postgresDBCluster
       TargetType: AWS::RDS::DBCluster
+
 Outputs:
-  postgreswebSecret: # injected as POSTGRESWEB_SECRET environment variable by Copilot.
+  postgresSecret:
     Description: "The JSON secret that holds the database username and password. Fields are 'host', 'port', 'dbname', 'username', 'password', 'dbClusterIdentifier' and 'engine'"
-    Value: !Ref postgreswebAuroraSecret
-  postgreswebSecurityGroup:
+    Value: !Ref postgresAuroraSecret
+    Export:
+      Name: !Sub ${App}-${Env}-postgresAuroraSecret
+  postgresSecurityGroup:
     Description: "The security group to attach to the workload."
-    Value: !Ref postgreswebSecurityGroup
+    Value: !Ref postgresWorkloadSecurityGroup
+    Export:
+      Name: !Sub ${App}-${Env}-postgresSecurityGroup

infra/copilot/web/addons/pems-db.yml renamed to infra/copilot/environments/addons/s3-web.yml

@@ -4,14 +4,12 @@ Parameters:
     Description: Your application's name.
   Env:
     Type: String
-    Description: The environment name your service, job, or workflow is being deployed to.
-  Name:
-    Type: String
-    Description: Your workload's name.
+    Description: The name of the environment being deployed.
+
 Resources:
-  pemsdbBucket:
+  s3webBucket:
     Metadata:
-      "aws:copilot:description": "An Amazon S3 bucket to store and retrieve objects for pems-db"
+      "aws:copilot:description": "An Amazon S3 bucket, s3-web, for storing and retrieving objects"
     Type: AWS::S3::Bucket
     Properties:
       VersioningConfiguration:
@@ -37,7 +35,7 @@ Resources:
             AbortIncompleteMultipartUpload:
               DaysAfterInitiation: 1
 
-  pemsdbBucketPolicy:
+  s3webBucketPolicy:
     Metadata:
       "aws:copilot:description": "A bucket policy to deny unencrypted access to the bucket and its contents"
     Type: AWS::S3::BucketPolicy
@@ -51,43 +49,21 @@ Resources:
             Principal: "*"
             Action: "s3:*"
             Resource:
-              - !Sub ${ pemsdbBucket.Arn}/*
-              - !Sub ${ pemsdbBucket.Arn}
+              - !Sub ${ s3webBucket.Arn}/*
+              - !Sub ${ s3webBucket.Arn}
             Condition:
               Bool:
                 "aws:SecureTransport": false
-      Bucket: !Ref pemsdbBucket
-
-  pemsdbAccessPolicy:
-    Metadata:
-      "aws:copilot:description": "An IAM ManagedPolicy for your service to access the pems-db bucket"
-    Type: AWS::IAM::ManagedPolicy
-    Properties:
-      Description: !Sub
-        - Grants CRUD access to the S3 bucket ${Bucket}
-        - { Bucket: !Ref pemsdbBucket }
-      PolicyDocument:
-        Version: "2012-10-17"
-        Statement:
-          - Sid: S3ObjectActions
-            Effect: Allow
-            Action:
-              - s3:GetObject
-              - s3:PutObject
-              - s3:PutObjectACL
-              - s3:PutObjectTagging
-              - s3:DeleteObject
-              - s3:RestoreObject
-            Resource: !Sub ${ pemsdbBucket.Arn}/*
-          - Sid: S3ListAction
-            Effect: Allow
-            Action: s3:ListBucket
-            Resource: !Sub ${ pemsdbBucket.Arn}
+      Bucket: !Ref s3webBucket
 
 Outputs:
-  pemsdbName:
+  s3webName:
     Description: "The name of a user-defined bucket."
-    Value: !Ref pemsdbBucket
-  pemsdbAccessPolicy:
-    Description: "The IAM::ManagedPolicy to attach to the task role"
-    Value: !Ref pemsdbAccessPolicy
+    Value: !Ref s3webBucket
+    Export:
+      Name: !Sub ${App}-${Env}-s3webBucketName
+  s3webBucketARN:
+    Description: "The ARN of the s3-web bucket."
+    Value: !GetAtt s3webBucket.Arn
+    Export:
+      Name: !Sub ${App}-${Env}-s3webBucketARN