From ffab34ac30a6f3bab8e1cbad868fb9cb0a0203f0 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Tue, 21 Oct 2025 11:12:23 -0500 Subject: [PATCH 01/37] Add json TD specification for production and staging, from AWS unmodified --- .../production.graphql-engine.ecs-td.json | 112 +++++++++++++++++ .../staging.graphql-engine.ecs-td.json | 117 ++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json create mode 100644 moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json diff --git a/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json new file mode 100644 index 0000000000..395bcfb100 --- /dev/null +++ b/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json @@ -0,0 +1,112 @@ +{ + "family": "atd-moped-production", + "containerDefinitions": [ + { + "cpu": 4096, + "environment": [ + { + "name": "HASURA_ENDPOINT", + "value": "https://moped-graphql.austinmobility.io/v1/graphql" + }, + { + "name": "HASURA_GRAPHQL_ENABLE_TELEMETRY", + "value": "false" + }, + { + "name": "HASURA_GRAPHQL_ENABLE_CONSOLE", + "value": "false" + } + ], + "essential": true, + "healthCheck": { + "command": [ + "CMD-SHELL", + "curl -fsS http://localhost:8080/healthz?strict=true || exit 1" + ], + "interval": 15, + "retries": 3, + "startPeriod": 15, + "timeout": 5 + }, + "image": "hasura/graphql-engine:v2.48.5", + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/aws/ecs/moped/graphql-api/production", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "ecs" + } + }, + "memory": 8192, + "mountPoints": [], + "name": "atd-moped", + "portMappings": [ + { + "containerPort": 8080, + "hostPort": 8080, + "name": "atd-moped-8080-tcp", + "protocol": "tcp" + } + ], + "secrets": [ + { + "name": "ACTIVITY_LOG_API_SECRET", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_PRODUCTION_FARGATE_HASURA_GRAPHQL_ADMIN_SECRET" + }, + { + "name": "HASURA_GRAPHQL_ADMIN_SECRET", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_PRODUCTION_FARGATE_HASURA_GRAPHQL_ADMIN_SECRET" + }, + { + "name": "HASURA_GRAPHQL_DATABASE_URL", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_PRODUCTION_FARGATE_HASURA_GRAPHQL_DATABASE_URL" + }, + { + "name": "HASURA_GRAPHQL_JWT_SECRET", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_PRODUCTION_FARGATE_HASURA_GRAPHQL_JWT_SECRET" + }, + { + "name": "MOPED_API_ACTIONS_URL", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_PRODUCTION_FARGATE_HASURA_ACTIONS_URL" + }, + { + "name": "MOPED_API_APIKEY", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_PRODUCTION_FARGATE_HASURA_API_KEY" + } + ], + "systemControls": [], + "volumesFrom": [] + } + ], + "executionRoleArn": "arn:aws:iam::295525487728:role/atd-moped-ecs-execution-role-production", + "networkMode": "awsvpc", + "volumes": [], + "placementConstraints": [], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "4096", + "memory": "8192", + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "tags": [ + { + "key": "project", + "value": "atd-moped" + }, + { + "key": "environment", + "value": "production" + }, + { + "key": "organization", + "value": "atd" + }, + { + "key": "awsApplication", + "value": "arn:aws:resource-groups:us-east-1:295525487728:group/Moped/0d1wuyh5kinejubg588qch09jg" + } + ] +} \ No newline at end of file diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json new file mode 100644 index 0000000000..649316e352 --- /dev/null +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -0,0 +1,117 @@ +{ + "family": "atd-moped-staging", + "containerDefinitions": [ + { + "cpu": 512, + "environment": [ + { + "name": "HASURA_ENDPOINT", + "value": "https://moped-graphql-staging.austinmobility.io/v1/graphql" + }, + { + "name": "HASURA_GRAPHQL_ENABLE_TELEMETRY", + "value": "false" + }, + { + "name": "HASURA_GRAPHQL_ENABLE_CONSOLE", + "value": "false" + } + ], + "essential": true, + "healthCheck": { + "command": [ + "CMD-SHELL", + "curl -fsS http://localhost:8080/healthz?strict=true || exit 1" + ], + "interval": 15, + "retries": 3, + "startPeriod": 15, + "timeout": 5 + }, + "image": "hasura/graphql-engine:v2.48.5", + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/aws/ecs/moped/graphql-api/staging", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "ecs" + } + }, + "memory": 2048, + "memoryReservation": 2048, + "mountPoints": [], + "name": "atd-moped", + "portMappings": [ + { + "containerPort": 8080, + "hostPort": 8080, + "name": "atd-moped-8080-tcp", + "protocol": "tcp" + } + ], + "secrets": [ + { + "name": "ACTIVITY_LOG_API_SECRET", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_STAGING_FARGATE_HASURA_GRAPHQL_ADMIN_SECRET" + }, + { + "name": "HASURA_GRAPHQL_ADMIN_SECRET", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_STAGING_FARGATE_HASURA_GRAPHQL_ADMIN_SECRET" + }, + { + "name": "HASURA_GRAPHQL_DATABASE_URL", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_STAGING_FARGATE_HASURA_GRAPHQL_DATABASE_URL" + }, + { + "name": "HASURA_GRAPHQL_JWT_SECRET", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_STAGING_FARGATE_HASURA_GRAPHQL_JWT_SECRET" + }, + { + "name": "MOPED_API_ACTIONS_URL", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_STAGING_FARGATE_HASURA_ACTIONS_URL" + }, + { + "name": "MOPED_API_APIKEY", + "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_STAGING_FARGATE_HASURA_API_KEY" + } + ], + "systemControls": [], + "volumesFrom": [] + } + ], + "executionRoleArn": "arn:aws:iam::295525487728:role/atd-moped-ecs-execution-role-staging", + "networkMode": "awsvpc", + "volumes": [], + "placementConstraints": [], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "2048", + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "tags": [ + { + "key": "project", + "value": "atd-moped" + }, + { + "key": "environment", + "value": "staging" + }, + { + "key": "ecs:taskDefinition:createdFrom", + "value": "ecs-console-v2" + }, + { + "key": "organization", + "value": "atd" + }, + { + "key": "awsApplication", + "value": "arn:aws:resource-groups:us-east-1:295525487728:group/Moped/0d1wuyh5kinejubg588qch09jg" + } + ] +} \ No newline at end of file From 32173cd2398d95169c930557e102c6b48a00a4bf Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Tue, 21 Oct 2025 11:45:53 -0500 Subject: [PATCH 02/37] Rename workflow and helper script, and enhance task names in job definition --- ... => migrations-metadata-application-helper.sh} | 4 ++-- ...ase.yml => migrations-metadata-deployment.yml} | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) rename .github/workflows/{aws-moped-migrations-helper.sh => migrations-metadata-application-helper.sh} (96%) rename .github/workflows/{atd_moped_database.yml => migrations-metadata-deployment.yml} (78%) diff --git a/.github/workflows/aws-moped-migrations-helper.sh b/.github/workflows/migrations-metadata-application-helper.sh similarity index 96% rename from .github/workflows/aws-moped-migrations-helper.sh rename to .github/workflows/migrations-metadata-application-helper.sh index 442ebbeb3a..49cdd13321 100644 --- a/.github/workflows/aws-moped-migrations-helper.sh +++ b/.github/workflows/migrations-metadata-application-helper.sh @@ -19,9 +19,9 @@ function download_hasura_settings() { echo "Downloading Hasura Settings: ${WORKING_STAGE}"; if [[ "${WORKING_STAGE}" == "production" ]]; then - export AWS_HASURA_CONFIGURATION="${AWS_MOPED_HASURA_CONFIGURAITON_FILE_PRODUCTION}"; + export AWS_HASURA_CONFIGURATION="${AWS_MOPED_HASURA_CONFIGURATION_FILE_PRODUCTION}"; else - export AWS_HASURA_CONFIGURATION="${AWS_MOPED_HASURA_CONFIGURAITON_FILE_STAGING}"; + export AWS_HASURA_CONFIGURATION="${AWS_MOPED_HASURA_CONFIGURATION_FILE_STAGING}"; fi; aws secretsmanager get-secret-value \ diff --git a/.github/workflows/atd_moped_database.yml b/.github/workflows/migrations-metadata-deployment.yml similarity index 78% rename from .github/workflows/atd_moped_database.yml rename to .github/workflows/migrations-metadata-deployment.yml index 8cacfa06ec..850171e02e 100644 --- a/.github/workflows/atd_moped_database.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -14,16 +14,17 @@ on: - ".github/workflows/aws-moped-migrations-helper.sh" jobs: - apply: - name: Apply Migrations + apply-migrations-and-metadata: + name: Apply Migrations and Metadata runs-on: ubuntu-24.04 steps: - uses: actions/setup-python@v4 + name: "Setup Python" with: python-version: "3.8" architecture: "x64" # Get the code first - - name: "Checkout" + - name: "Checkout Code" uses: actions/checkout@v3 # Then install the AWC CLI tools & boto3 - name: "Install AWS Cli & Hasura CLI" @@ -33,18 +34,18 @@ jobs: curl -L https://github.com/hasura/graphql-engine/raw/stable/cli/get.sh | bash # Run the shell commands using the AWS environment variables - - name: "Apply" + - name: "Apply Migrations and Metadata" env: AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_MOPED_HASURA_CONFIGURAITON_FILE_PRODUCTION: ${{ secrets.AWS_MOPED_HASURA_CONFIGURAITON_FILE_PRODUCTION }} - AWS_MOPED_HASURA_CONFIGURAITON_FILE_STAGING: ${{ secrets.AWS_MOPED_HASURA_CONFIGURAITON_FILE_STAGING }} + AWS_MOPED_HASURA_CONFIGURATION_FILE_PRODUCTION: ${{ secrets.AWS_MOPED_HASURA_CONFIGURAITON_FILE_PRODUCTION }} + AWS_MOPED_HASURA_CONFIGURATION_FILE_STAGING: ${{ secrets.AWS_MOPED_HASURA_CONFIGURAITON_FILE_STAGING }} run: | export BRANCH_NAME=${GITHUB_REF##*/} echo "SHA: ${GITHUB_SHA}" echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}" echo "GR: ${GITHUB_REF}" echo "PWD: $(pwd)" - source $(pwd)/.github/workflows/aws-moped-migrations-helper.sh + source $(pwd)/.github/workflows/migration-metadata-application-helper.sh run_migration_process From 21896fba9c83aab99a42344f08682ffb72501463 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 22 Oct 2025 10:26:54 -0500 Subject: [PATCH 03/37] Add placeholder job to migration metadata workflow for completion notification --- .github/workflows/migrations-metadata-deployment.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 850171e02e..9ccfb82a4a 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -49,3 +49,12 @@ jobs: echo "PWD: $(pwd)" source $(pwd)/.github/workflows/migration-metadata-application-helper.sh run_migration_process + + placeholder-job: + name: Placeholder Job + runs-on: ubuntu-24.04 + needs: apply-migrations-and-metadata + if: always() + steps: + - name: "Print Message" + run: echo "Migrations and metadata job has completed" From daee94568121da9b768a250785d6700108ff701a Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 23 Oct 2025 11:51:21 -0500 Subject: [PATCH 04/37] rename this file to match --- ...ication-helper.sh => migrations-metadata-deployment-helper.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{migrations-metadata-application-helper.sh => migrations-metadata-deployment-helper.sh} (100%) diff --git a/.github/workflows/migrations-metadata-application-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh similarity index 100% rename from .github/workflows/migrations-metadata-application-helper.sh rename to .github/workflows/migrations-metadata-deployment-helper.sh From 6a120da29ebb30b488b759489d17e4736bf82ad5 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 23 Oct 2025 11:51:35 -0500 Subject: [PATCH 05/37] remove header; there is a `name` directly below it with the same content --- .github/workflows/migrations-metadata-deployment.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 9ccfb82a4a..6cdc38ebed 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -1,6 +1,3 @@ -# -# Applies database migrations to staging (main) and production -# name: "Applies the migrations to the database" on: @@ -47,7 +44,7 @@ jobs: echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}" echo "GR: ${GITHUB_REF}" echo "PWD: $(pwd)" - source $(pwd)/.github/workflows/migration-metadata-application-helper.sh + source $(pwd)/.github/workflows/migration-metadata-deployment-helper.sh run_migration_process placeholder-job: From 8bc92adb07424c87f8f2e135d4b50ad8f5aa4809 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 23 Oct 2025 11:55:19 -0500 Subject: [PATCH 06/37] disarm the migration job for development --- .github/workflows/migrations-metadata-deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 6cdc38ebed..e95cd0f2fc 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -45,7 +45,7 @@ jobs: echo "GR: ${GITHUB_REF}" echo "PWD: $(pwd)" source $(pwd)/.github/workflows/migration-metadata-deployment-helper.sh - run_migration_process + # run_migration_process placeholder-job: name: Placeholder Job From 3489c5f4cfcca65c78198ad7465e33c8599c77ec Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 23 Oct 2025 11:56:15 -0500 Subject: [PATCH 07/37] Update path for triggering files --- .github/workflows/migrations-metadata-deployment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index e95cd0f2fc..0ca9c9e24f 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -7,8 +7,8 @@ on: - production paths: - "moped-database/**" - - ".github/workflows/atd_moped_database.yml" - - ".github/workflows/aws-moped-migrations-helper.sh" + - ".github/workflows/migrations-metadata-deployment.yml" + - ".github/workflows/migrations-metadata-deployment-helper.sh" jobs: apply-migrations-and-metadata: From e6fd5e582f43bd1ff9143a80292bd948e7cdb9c6 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 23 Oct 2025 12:00:44 -0500 Subject: [PATCH 08/37] Start to address boilerplate code --- .github/workflows/migrations-metadata-deployment.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 0ca9c9e24f..6648aca806 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -47,11 +47,11 @@ jobs: source $(pwd)/.github/workflows/migration-metadata-deployment-helper.sh # run_migration_process - placeholder-job: - name: Placeholder Job + update-ecs-task-deployment: + name: Check for updated ECS task definition and deploy if needed runs-on: ubuntu-24.04 needs: apply-migrations-and-metadata if: always() steps: - - name: "Print Message" - run: echo "Migrations and metadata job has completed" + - name: "Job stub: Update ECS task definition and deploy if needed" + run: echo "We're in the update-ecs-task-deployment job" From e3efa7011ed1ebee351031df71556d7ed61cf340 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 10:28:07 -0500 Subject: [PATCH 09/37] allow for this branch to trigger actions for dev --- .github/workflows/migrations-metadata-deployment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 6648aca806..34e2861412 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -5,6 +5,7 @@ on: branches: - main - production + - frank/version-control-ecs-task-definitions paths: - "moped-database/**" - ".github/workflows/migrations-metadata-deployment.yml" From 19b9a5b10799919f9b506673cc77c0c105e7d2cf Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 10:31:37 -0500 Subject: [PATCH 10/37] typo in file name --- .github/workflows/migrations-metadata-deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 34e2861412..bb6459588b 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -45,7 +45,7 @@ jobs: echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}" echo "GR: ${GITHUB_REF}" echo "PWD: $(pwd)" - source $(pwd)/.github/workflows/migration-metadata-deployment-helper.sh + source $(pwd)/.github/workflows/migrations-metadata-deployment-helper.sh # run_migration_process update-ecs-task-deployment: From 9623be3b9613e391887914d4b5f1318796cc92de Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 11:00:08 -0500 Subject: [PATCH 11/37] Add ECS task definition validation and update process to deployment workflow - Introduced functions for determining and validating ECS task definitions based on the branch name. - Enhanced the migration metadata deployment workflow to include steps for checking out code, configuring AWS credentials, and validating the ECS task definition. - Updated comments for clarity and organization in the helper script. --- .../migrations-metadata-deployment-helper.sh | 74 ++++++++++++++++++- .../migrations-metadata-deployment.yml | 25 ++++++- 2 files changed, 93 insertions(+), 6 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index 49cdd13321..2c965b0310 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -12,8 +12,13 @@ esac echo "SOURCE -> BRANCH_NAME: ${BRANCH_NAME}" echo "SOURCE -> WORKING_STAGE: ${WORKING_STAGE}" +################################################################################ +# JOB: apply-migrations-and-metadata +# Functions for downloading Hasura settings and applying migrations/metadata +################################################################################ + # -# Download the Zappa settings from the AWS Secrets Manager +# Download the Hasura settings from the AWS Secrets Manager # function download_hasura_settings() { echo "Downloading Hasura Settings: ${WORKING_STAGE}"; @@ -29,9 +34,8 @@ function download_hasura_settings() { jq -rc ".SecretString" > config.yaml; } - # -# Waits until the local hasura server is ready +# Applies migrations and metadata to the Hasura instance # function run_migration() { echo "----- MIGRATIONS STARTED -----"; @@ -48,7 +52,7 @@ function run_migration() { } # -# Controls the migration process +# Controls the migration process (main entry point for migrations job) # function run_migration_process() { cd ./moped-database; @@ -56,3 +60,65 @@ function run_migration_process() { download_hasura_settings; run_migration; } + +################################################################################ +# JOB: update-ecs-task-deployment +# Functions for validating and deploying ECS task definitions +################################################################################ + +# +# Determines which ECS task definition file to use based on the branch +# Sets TD_FILE and ENVIRONMENT variables +# +function determine_task_definition_file() { + echo "Branch name: ${BRANCH_NAME}"; + + # Determine environment based on branch + if [ "${BRANCH_NAME}" = "production" ]; then + export ENVIRONMENT="production" + export TD_FILE="moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json" + else + export ENVIRONMENT="staging" + export TD_FILE="moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json" + fi + + echo "Environment: ${ENVIRONMENT}"; + echo "Task definition file: ${TD_FILE}"; +} + +# +# Validates the ECS task definition JSON file using AWS CLI +# Returns 0 if valid, exits with error if invalid or file not found +# +function validate_task_definition() { + # Check if task definition file exists + if [ ! -f "${TD_FILE}" ]; then + echo "Task definition file not found: ${TD_FILE}"; + echo "Skipping ECS task definition update"; + exit 0; + fi + + echo "Task definition file found, validating..."; + + # Validate the task definition using AWS CLI + # This will check if the JSON is valid and compatible with ECS + # The --dry-run flag ensures we don't actually register the task definition + if aws ecs register-task-definition \ + --cli-input-json file://${TD_FILE} \ + --dry-run; then + echo "✓ Task definition is valid!"; + return 0; + else + echo "✗ Task definition validation failed!"; + exit 1; + fi +} + +# +# Main entry point for ECS task definition update process +# Determines the correct file and validates it +# +function update_ecs_task_definition_process() { + determine_task_definition_file; + validate_task_definition; +} diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index bb6459588b..1c8d9d76bf 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -54,5 +54,26 @@ jobs: needs: apply-migrations-and-metadata if: always() steps: - - name: "Job stub: Update ECS task definition and deploy if needed" - run: echo "We're in the update-ecs-task-deployment job" + - name: "Checkout Code" + uses: actions/checkout@v3 + + - name: "Configure AWS credentials" + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: "Validate ECS task definition" + env: + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + export BRANCH_NAME=${GITHUB_REF##*/} + echo "SHA: ${GITHUB_SHA}" + echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}" + echo "GR: ${GITHUB_REF}" + echo "PWD: $(pwd)" + source $(pwd)/.github/workflows/migrations-metadata-deployment-helper.sh + update_ecs_task_definition_process From 5b879d665bbb17c228f8815f283986215230fccb Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 11:02:19 -0500 Subject: [PATCH 12/37] Add AWS cli tooling --- .github/workflows/migrations-metadata-deployment.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 1c8d9d76bf..92c156132c 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -57,6 +57,12 @@ jobs: - name: "Checkout Code" uses: actions/checkout@v3 + - name: "Install AWS CLI" + run: | + sudo apt-get update + sudo apt-get install -y python3-setuptools + pip3 install awscli + - name: "Configure AWS credentials" uses: aws-actions/configure-aws-credentials@v2 with: From ecdaca6f1ed43b7d5ad1330d4fb518f05daa363c Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 11:05:11 -0500 Subject: [PATCH 13/37] Use official installation instructions from AWS for cli --- .github/workflows/migrations-metadata-deployment.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 92c156132c..12c1694e2a 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -57,11 +57,12 @@ jobs: - name: "Checkout Code" uses: actions/checkout@v3 - - name: "Install AWS CLI" + - name: "Install AWS CLI v2" run: | - sudo apt-get update - sudo apt-get install -y python3-setuptools - pip3 install awscli + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install + aws --version - name: "Configure AWS credentials" uses: aws-actions/configure-aws-credentials@v2 From f02650442ca2f0d22bed3f4a2c78a36bb4612c0c Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 11:09:46 -0500 Subject: [PATCH 14/37] V2 out of the ubuntu repo --- .../workflows/migrations-metadata-deployment.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 12c1694e2a..21f77e50ac 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -57,12 +57,17 @@ jobs: - name: "Checkout Code" uses: actions/checkout@v3 - - name: "Install AWS CLI v2" + # - name: "Install AWS CLI v2" + # run: | + # curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + # unzip awscliv2.zip + # sudo ./aws/install + # aws --version + + - name: "Install AWS CLI" run: | - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - sudo ./aws/install - aws --version + sudo apt-get update + sudo apt-get install -y awscli - name: "Configure AWS credentials" uses: aws-actions/configure-aws-credentials@v2 From 43d88f71d1519b27b357788f587b8989ff1579a3 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 11:19:13 -0500 Subject: [PATCH 15/37] Update AWS CLI installation method in deployment workflow - Replaced the previous AWS CLI installation method with the official installation instructions for AWS CLI v2. - Commented out the old installation and configuration steps for clarity and future reference. --- .../migrations-metadata-deployment.yml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 21f77e50ac..ab6aa79d01 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -57,24 +57,24 @@ jobs: - name: "Checkout Code" uses: actions/checkout@v3 - # - name: "Install AWS CLI v2" - # run: | - # curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - # unzip awscliv2.zip - # sudo ./aws/install - # aws --version - - - name: "Install AWS CLI" + - name: "Install AWS CLI v2" run: | - sudo apt-get update - sudo apt-get install -y awscli + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install --update + aws --version - - name: "Configure AWS credentials" - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + # - name: "Install AWS CLI" + # run: | + # sudo apt-get update + # sudo apt-get install -y awscli + + # - name: "Configure AWS credentials" + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - name: "Validate ECS task definition" env: From c036b93b3dcc261f8668e52b2c6c76f03bbbcd90 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 11:35:05 -0500 Subject: [PATCH 16/37] Refactor ECS task definition handling in deployment workflow - Renamed the function for validating ECS task definitions to better reflect its purpose, changing it to register the task definition. - Updated the logic to register the ECS task definition using AWS CLI instead of validating it. - Adjusted comments for clarity regarding the registration process and its success or failure. --- .../migrations-metadata-deployment-helper.sh | 24 +++++++++---------- .../migrations-metadata-deployment.yml | 6 +---- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index 2c965b0310..c938775f29 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -87,10 +87,10 @@ function determine_task_definition_file() { } # -# Validates the ECS task definition JSON file using AWS CLI -# Returns 0 if valid, exits with error if invalid or file not found +# Registers the ECS task definition using AWS CLI +# Returns 0 if successful, exits with error if registration fails or file not found # -function validate_task_definition() { +function register_task_definition() { # Check if task definition file exists if [ ! -f "${TD_FILE}" ]; then echo "Task definition file not found: ${TD_FILE}"; @@ -98,27 +98,25 @@ function validate_task_definition() { exit 0; fi - echo "Task definition file found, validating..."; + echo "Task definition file found, registering..."; - # Validate the task definition using AWS CLI - # This will check if the JSON is valid and compatible with ECS - # The --dry-run flag ensures we don't actually register the task definition + # Register the task definition using AWS CLI if aws ecs register-task-definition \ - --cli-input-json file://${TD_FILE} \ - --dry-run; then - echo "✓ Task definition is valid!"; + --family ${FAMILY} \ + --cli-input-json file://${TD_FILE}; then + echo "✓ Task definition registered successfully!"; return 0; else - echo "✗ Task definition validation failed!"; + echo "✗ Task definition registration failed!"; exit 1; fi } # # Main entry point for ECS task definition update process -# Determines the correct file and validates it +# Determines the correct file and registers it # function update_ecs_task_definition_process() { determine_task_definition_file; - validate_task_definition; + register_task_definition; } diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index ab6aa79d01..8636f19f34 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -28,6 +28,7 @@ jobs: - name: "Install AWS Cli & Hasura CLI" run: | sudo apt-get install python3-setuptools + # reminder; this is V1 of the AWS CLI pip3 install awscli virtualenv curl -L https://github.com/hasura/graphql-engine/raw/stable/cli/get.sh | bash @@ -64,11 +65,6 @@ jobs: sudo ./aws/install --update aws --version - # - name: "Install AWS CLI" - # run: | - # sudo apt-get update - # sudo apt-get install -y awscli - # - name: "Configure AWS credentials" # uses: aws-actions/configure-aws-credentials@v2 # with: From bb46adf9dd39742653976c67da1201098870c3e7 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 11:37:18 -0500 Subject: [PATCH 17/37] Add ECS task family environment variable to deployment helper script --- .github/workflows/migrations-metadata-deployment-helper.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index c938775f29..e6d3b96a3f 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -77,9 +77,11 @@ function determine_task_definition_file() { if [ "${BRANCH_NAME}" = "production" ]; then export ENVIRONMENT="production" export TD_FILE="moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json" + export FAMILY="atd-moped-production" else export ENVIRONMENT="staging" export TD_FILE="moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json" + export FAMILY="atd-moped-staging" fi echo "Environment: ${ENVIRONMENT}"; From 23c1ceaf14a8f21faed01f39e88e639415ee1209 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 13:33:02 -0500 Subject: [PATCH 18/37] Checking if file changed in this push --- .../migrations-metadata-deployment-helper.sh | 29 +++++++++++++++++++ .../migrations-metadata-deployment.yml | 9 +----- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index e6d3b96a3f..12c15f23f6 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -88,6 +88,29 @@ function determine_task_definition_file() { echo "Task definition file: ${TD_FILE}"; } +# +# Checks if the task definition file was modified in this push +# Returns 0 if changed, 1 if unchanged +# +function check_task_definition_changed() { + echo "Checking if task definition file changed in this push..."; + + # Check if this is the first commit (no previous commit to compare) + if ! git rev-parse HEAD~1 >/dev/null 2>&1; then + echo "First commit detected, will register task definition"; + return 0; + fi + + # Check if the task definition file was modified in the current commit + if git diff --name-only HEAD~1 HEAD | grep -q "^${TD_FILE}$"; then + echo "✓ Task definition file has changed"; + return 0; + else + echo "Task definition file unchanged, skipping registration"; + return 1; + fi +} + # # Registers the ECS task definition using AWS CLI # Returns 0 if successful, exits with error if registration fails or file not found @@ -100,6 +123,12 @@ function register_task_definition() { exit 0; fi + # Check if the task definition file changed in this push + if ! check_task_definition_changed; then + echo "Skipping ECS task definition registration"; + exit 0; + fi + echo "Task definition file found, registering..."; # Register the task definition using AWS CLI diff --git a/.github/workflows/migrations-metadata-deployment.yml b/.github/workflows/migrations-metadata-deployment.yml index 8636f19f34..a3c77d2ba8 100644 --- a/.github/workflows/migrations-metadata-deployment.yml +++ b/.github/workflows/migrations-metadata-deployment.yml @@ -65,14 +65,7 @@ jobs: sudo ./aws/install --update aws --version - # - name: "Configure AWS credentials" - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - - name: "Validate ECS task definition" + - name: "Install updated ECS task definition" env: AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} From 337196f62e86009b9b55012b5ae3cde6481ef6e1 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 13:35:11 -0500 Subject: [PATCH 19/37] Sync up to latest graphql-engine --- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index 649316e352..050d75d297 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -28,7 +28,7 @@ "startPeriod": 15, "timeout": 5 }, - "image": "hasura/graphql-engine:v2.48.5", + "image": "hasura/graphql-engine:v2.48.6", "logConfiguration": { "logDriver": "awslogs", "options": { From 8b5435224ee6637749250d6190702528a64ece36 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 13:46:40 -0500 Subject: [PATCH 20/37] try a more robust approach to change detection --- .../migrations-metadata-deployment-helper.sh | 57 +++++++++++++------ 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index 12c15f23f6..671f7d3731 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -89,25 +89,48 @@ function determine_task_definition_file() { } # -# Checks if the task definition file was modified in this push -# Returns 0 if changed, 1 if unchanged +# Compares the local task definition file with the one currently in AWS +# Returns 0 if different (needs update), 1 if identical (no update needed) # -function check_task_definition_changed() { - echo "Checking if task definition file changed in this push..."; - - # Check if this is the first commit (no previous commit to compare) - if ! git rev-parse HEAD~1 >/dev/null 2>&1; then - echo "First commit detected, will register task definition"; +function check_task_definition_differs() { + echo "Fetching current task definition from AWS for family: ${FAMILY}..."; + + # Describe the current task definition from AWS + # If this is the first task definition, the command will fail and we'll register it + if ! aws ecs describe-task-definition \ + --task-definition ${FAMILY} \ + --output json > /tmp/aws-task-def.json 2>/dev/null; then + echo "No existing task definition found in AWS, will register new one"; return 0; fi - # Check if the task definition file was modified in the current commit - if git diff --name-only HEAD~1 HEAD | grep -q "^${TD_FILE}$"; then - echo "✓ Task definition file has changed"; - return 0; - else - echo "Task definition file unchanged, skipping registration"; + echo "Extracting task definition from AWS response..."; + + # Extract just the taskDefinition object and remove AWS-managed fields + # These fields are added by AWS and shouldn't be compared + jq --sort-keys '.taskDefinition | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt)' \ + /tmp/aws-task-def.json > /tmp/aws-task-def-normalized.json + + # Normalize the local file the same way (remove the same fields if present) + jq --sort-keys 'del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt)' \ + ${TD_FILE} > /tmp/local-task-def-normalized.json + + echo "Comparing local task definition with AWS version..."; + + # Compare the normalized JSON files + if diff -q /tmp/aws-task-def-normalized.json /tmp/local-task-def-normalized.json > /dev/null; then + echo "Task definitions are identical, no update needed"; return 1; + else + echo "✓ Task definitions differ, update needed"; + echo ""; + echo "========================================"; + echo "Differences (AWS version vs Local file):"; + echo "========================================"; + diff -u /tmp/aws-task-def-normalized.json /tmp/local-task-def-normalized.json || true + echo "========================================"; + echo ""; + return 0; fi } @@ -123,13 +146,13 @@ function register_task_definition() { exit 0; fi - # Check if the task definition file changed in this push - if ! check_task_definition_changed; then + # Check if the task definition differs from what's in AWS + if ! check_task_definition_differs; then echo "Skipping ECS task definition registration"; exit 0; fi - echo "Task definition file found, registering..."; + echo "Registering updated task definition..."; # Register the task definition using AWS CLI if aws ecs register-task-definition \ From 1828ec6cb9712934e8984d17e3b95652600912fd Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 13:50:33 -0500 Subject: [PATCH 21/37] Update workflow, but not TD, to test normalization technique --- .github/workflows/migrations-metadata-deployment-helper.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index 671f7d3731..dda3452c2c 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -108,11 +108,11 @@ function check_task_definition_differs() { # Extract just the taskDefinition object and remove AWS-managed fields # These fields are added by AWS and shouldn't be compared - jq --sort-keys '.taskDefinition | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt)' \ + jq --sort-keys '.taskDefinition | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt, .tags)' \ /tmp/aws-task-def.json > /tmp/aws-task-def-normalized.json # Normalize the local file the same way (remove the same fields if present) - jq --sort-keys 'del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt)' \ + jq --sort-keys 'del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt, .tags)' \ ${TD_FILE} > /tmp/local-task-def-normalized.json echo "Comparing local task definition with AWS version..."; From ce0a54b4e1188fdc02904dee30e8333563c9cad0 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 13:56:40 -0500 Subject: [PATCH 22/37] OK, this may be working -- bump the ram for example --- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index 050d75d297..11136d25f2 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -87,7 +87,7 @@ "FARGATE" ], "cpu": "512", - "memory": "2048", + "memory": "4096", "runtimePlatform": { "cpuArchitecture": "X86_64", "operatingSystemFamily": "LINUX" From 78b49b1d92dfb5e6ac350e6874e93f8c8caec948 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 14:02:09 -0500 Subject: [PATCH 23/37] Add deregistration support --- .../migrations-metadata-deployment-helper.sh | 61 +++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index dda3452c2c..5346f85d21 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -134,22 +134,71 @@ function check_task_definition_differs() { fi } +# +# Deregisters old task definitions, keeping only the last 3 active ones +# +function cleanup_old_task_definitions() { + echo "Cleaning up old task definitions for family: ${FAMILY}..."; + + # List all ACTIVE task definition revisions for this family, sorted by revision number + local active_revisions=$(aws ecs list-task-definitions \ + --family-prefix ${FAMILY} \ + --status ACTIVE \ + --sort DESC \ + --output json | jq -r '.taskDefinitionArns[]') + + if [ -z "$active_revisions" ]; then + echo "No active task definitions found to clean up"; + return 0; + fi + + # Count total active revisions + local total_count=$(echo "$active_revisions" | wc -l | tr -d ' ') + echo "Found ${total_count} active task definition(s)"; + + # If we have 3 or fewer, no cleanup needed + if [ "$total_count" -le 3 ]; then + echo "Only ${total_count} active revision(s), no cleanup needed"; + return 0; + fi + + # Skip the first 3 (most recent) and deregister the rest + local to_deregister=$(echo "$active_revisions" | tail -n +4) + local deregister_count=$(echo "$to_deregister" | wc -l | tr -d ' ') + + echo "Deregistering ${deregister_count} old revision(s), keeping the 3 most recent..."; + + while IFS= read -r task_def_arn; do + if [ -n "$task_def_arn" ]; then + echo "Deregistering: ${task_def_arn}"; + if aws ecs deregister-task-definition --task-definition "${task_def_arn}" > /dev/null; then + echo " ✓ Deregistered successfully"; + else + echo " ✗ Failed to deregister"; + fi + fi + done <<< "$to_deregister" + + echo "Cleanup complete!"; +} + # # Registers the ECS task definition using AWS CLI # Returns 0 if successful, exits with error if registration fails or file not found +# Returns 1 if no registration was needed (file unchanged) # function register_task_definition() { # Check if task definition file exists if [ ! -f "${TD_FILE}" ]; then echo "Task definition file not found: ${TD_FILE}"; echo "Skipping ECS task definition update"; - exit 0; + return 1; fi # Check if the task definition differs from what's in AWS if ! check_task_definition_differs; then echo "Skipping ECS task definition registration"; - exit 0; + return 1; fi echo "Registering updated task definition..."; @@ -168,9 +217,13 @@ function register_task_definition() { # # Main entry point for ECS task definition update process -# Determines the correct file and registers it +# Determines the correct file, registers it if needed, and cleans up old revisions # function update_ecs_task_definition_process() { determine_task_definition_file; - register_task_definition; + + # Only run cleanup if we successfully registered a new task definition + if register_task_definition; then + cleanup_old_task_definitions; + fi } From 8fc4d06cbd075000beeaae48896a1efee9a50a2a Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 14:09:02 -0500 Subject: [PATCH 24/37] we need to make sure we're running the latest before we deregister anything --- .github/workflows/migrations-metadata-deployment-helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index 5346f85d21..f354f9ec72 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -224,6 +224,6 @@ function update_ecs_task_definition_process() { # Only run cleanup if we successfully registered a new task definition if register_task_definition; then - cleanup_old_task_definitions; + # cleanup_old_task_definitions; fi } From d071d512124448f46ea16408bdfb6aef89197974 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 14:25:29 -0500 Subject: [PATCH 25/37] Add ECS service update functionality to deployment helper script - Introduced a new function to update the ECS service with the latest task definition. - Added environment variables for the ECS cluster and service. - Enhanced logging to provide more context during the update process. - Updated the main task definition update process to include the service update step. --- .../migrations-metadata-deployment-helper.sh | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index f354f9ec72..c04f2bcbd7 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -78,13 +78,20 @@ function determine_task_definition_file() { export ENVIRONMENT="production" export TD_FILE="moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json" export FAMILY="atd-moped-production" + export CLUSTER="atd-moped-cluster-production" else export ENVIRONMENT="staging" export TD_FILE="moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json" export FAMILY="atd-moped-staging" + export CLUSTER="atd-moped-cluster-staging" fi + export SERVICE="graphql-engine" + echo "Environment: ${ENVIRONMENT}"; + echo "Cluster: ${CLUSTER}"; + echo "Service: ${SERVICE}"; + echo "Family: ${FAMILY}"; echo "Task definition file: ${TD_FILE}"; } @@ -215,6 +222,30 @@ function register_task_definition() { fi } +# +# Updates the ECS service to use the latest task definition +# +function update_ecs_service() { + echo "Updating ECS service to use the new task definition..."; + echo "Cluster: ${CLUSTER}"; + echo "Service: ${SERVICE}"; + echo "Family: ${FAMILY}"; + + # Update the service to use the latest task definition from the family + if aws ecs update-service \ + --cluster ${CLUSTER} \ + --service ${SERVICE} \ + --task-definition ${FAMILY} \ + --force-new-deployment; then + echo "✓ ECS service updated successfully!"; + echo "The service will now use the new task definition"; + return 0; + else + echo "✗ Failed to update ECS service"; + exit 1; + fi +} + # # Main entry point for ECS task definition update process # Determines the correct file, registers it if needed, and cleans up old revisions @@ -222,8 +253,9 @@ function register_task_definition() { function update_ecs_task_definition_process() { determine_task_definition_file; - # Only run cleanup if we successfully registered a new task definition + # Only run cleanup and service update if we successfully registered a new task definition if register_task_definition; then - # cleanup_old_task_definitions; + # cleanup_old_task_definitions; # leave this commented out until I put it back in. + update_ecs_service; fi } From 91ce1f652d6bab0b69237203bd53a9542dd34d6d Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 14:26:43 -0500 Subject: [PATCH 26/37] Restore previous 2G memory allocation --- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index 11136d25f2..649316e352 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -28,7 +28,7 @@ "startPeriod": 15, "timeout": 5 }, - "image": "hasura/graphql-engine:v2.48.6", + "image": "hasura/graphql-engine:v2.48.5", "logConfiguration": { "logDriver": "awslogs", "options": { @@ -87,7 +87,7 @@ "FARGATE" ], "cpu": "512", - "memory": "4096", + "memory": "2048", "runtimePlatform": { "cpuArchitecture": "X86_64", "operatingSystemFamily": "LINUX" From 67105479ef3ce3896fdccd15dcf41ba4af881736 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 14:56:43 -0500 Subject: [PATCH 27/37] Allow deregistration of old TDs now --- .github/workflows/migrations-metadata-deployment-helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index c04f2bcbd7..8e0b22fd5b 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -255,7 +255,7 @@ function update_ecs_task_definition_process() { # Only run cleanup and service update if we successfully registered a new task definition if register_task_definition; then - # cleanup_old_task_definitions; # leave this commented out until I put it back in. + cleanup_old_task_definitions; update_ecs_service; fi } From 1bc63447e8dd8889f4f3656c8cbdfd92a158f440 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 14:58:56 -0500 Subject: [PATCH 28/37] Enhance logging in deployment helper script to indicate task definition comparison results with emojis --- .github/workflows/migrations-metadata-deployment-helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/migrations-metadata-deployment-helper.sh b/.github/workflows/migrations-metadata-deployment-helper.sh index 8e0b22fd5b..dea1855f56 100644 --- a/.github/workflows/migrations-metadata-deployment-helper.sh +++ b/.github/workflows/migrations-metadata-deployment-helper.sh @@ -126,7 +126,7 @@ function check_task_definition_differs() { # Compare the normalized JSON files if diff -q /tmp/aws-task-def-normalized.json /tmp/local-task-def-normalized.json > /dev/null; then - echo "Task definitions are identical, no update needed"; + echo "🛑 Task definitions are identical, no update needed"; return 1; else echo "✓ Task definitions differ, update needed"; From fc876083890ab6efab44c76918a7e2cfa046a1c9 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 15:02:41 -0500 Subject: [PATCH 29/37] Reduce CPU allocation in ECS task definition from 512 to 256 --- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index 649316e352..b7b319141e 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -86,7 +86,7 @@ "requiresCompatibilities": [ "FARGATE" ], - "cpu": "512", + "cpu": "256", "memory": "2048", "runtimePlatform": { "cpuArchitecture": "X86_64", From c4ea1abca3e65e8adba2494234c5726ea56fd2bd Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 15:04:09 -0500 Subject: [PATCH 30/37] Got a free test of invalid TD's on that one, worked good! --- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index b7b319141e..fed82a34db 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -2,7 +2,7 @@ "family": "atd-moped-staging", "containerDefinitions": [ { - "cpu": 512, + "cpu": 256, "environment": [ { "name": "HASURA_ENDPOINT", From d21cc061ae4a86fb646e6c1fe6b584b5c497d8d9 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 15:29:36 -0500 Subject: [PATCH 31/37] TD EOF Newlines --- .../production.graphql-engine.ecs-td.json | 12 ++++++------ .../staging.graphql-engine.ecs-td.json | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json index 395bcfb100..bfc31ae340 100644 --- a/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json @@ -38,7 +38,7 @@ } }, "memory": 8192, - "mountPoints": [], + "mountPoints": [ ], "name": "atd-moped", "portMappings": [ { @@ -74,14 +74,14 @@ "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_PRODUCTION_FARGATE_HASURA_API_KEY" } ], - "systemControls": [], - "volumesFrom": [] + "systemControls": [ ], + "volumesFrom": [ ] } ], "executionRoleArn": "arn:aws:iam::295525487728:role/atd-moped-ecs-execution-role-production", "networkMode": "awsvpc", - "volumes": [], - "placementConstraints": [], + "volumes": [ ], + "placementConstraints": [ ], "requiresCompatibilities": [ "FARGATE" ], @@ -109,4 +109,4 @@ "value": "arn:aws:resource-groups:us-east-1:295525487728:group/Moped/0d1wuyh5kinejubg588qch09jg" } ] -} \ No newline at end of file +} diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index fed82a34db..1cd95742ad 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -39,7 +39,7 @@ }, "memory": 2048, "memoryReservation": 2048, - "mountPoints": [], + "mountPoints": [ ], "name": "atd-moped", "portMappings": [ { @@ -75,14 +75,14 @@ "valueFrom": "arn:aws:ssm:us-east-1:295525487728:parameter/MOPED_STAGING_FARGATE_HASURA_API_KEY" } ], - "systemControls": [], - "volumesFrom": [] + "systemControls": [ ], + "volumesFrom": [ ] } ], "executionRoleArn": "arn:aws:iam::295525487728:role/atd-moped-ecs-execution-role-staging", "networkMode": "awsvpc", - "volumes": [], - "placementConstraints": [], + "volumes": [ ], + "placementConstraints": [ ], "requiresCompatibilities": [ "FARGATE" ], @@ -114,4 +114,4 @@ "value": "arn:aws:resource-groups:us-east-1:295525487728:group/Moped/0d1wuyh5kinejubg588qch09jg" } ] -} \ No newline at end of file +} From 1f172a6c453f9edb30ffc4c08354a85a555b72ed Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 27 Oct 2025 15:59:19 -0500 Subject: [PATCH 32/37] Get the version right now that dev is done --- .../ecs_task_definitions/production.graphql-engine.ecs-td.json | 2 +- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json index bfc31ae340..ae1f256f2c 100644 --- a/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/production.graphql-engine.ecs-td.json @@ -28,7 +28,7 @@ "startPeriod": 15, "timeout": 5 }, - "image": "hasura/graphql-engine:v2.48.5", + "image": "hasura/graphql-engine:v2.48.6", "logConfiguration": { "logDriver": "awslogs", "options": { diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index 1cd95742ad..13f6433e54 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -28,7 +28,7 @@ "startPeriod": 15, "timeout": 5 }, - "image": "hasura/graphql-engine:v2.48.5", + "image": "hasura/graphql-engine:v2.48.6", "logConfiguration": { "logDriver": "awslogs", "options": { From ce31eea5255717a93b12b113381f2e5b5ee1aa35 Mon Sep 17 00:00:00 2001 From: John Clary Date: Thu, 30 Oct 2025 13:31:29 -0400 Subject: [PATCH 33/37] john testing: bump memory and memoryReservation to 4096 --- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index 13f6433e54..df9e45ecdb 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -37,8 +37,8 @@ "awslogs-stream-prefix": "ecs" } }, - "memory": 2048, - "memoryReservation": 2048, + "memory": 4096, + "memoryReservation": 4096, "mountPoints": [ ], "name": "atd-moped", "portMappings": [ From dea7f4d7b92444af3b5d02c37fb49372e4a57052 Mon Sep 17 00:00:00 2001 From: John Clary Date: Thu, 30 Oct 2025 13:33:40 -0400 Subject: [PATCH 34/37] john testing: bump task memory to 4096 --- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index df9e45ecdb..1b4c32ca35 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -87,7 +87,7 @@ "FARGATE" ], "cpu": "256", - "memory": "2048", + "memory": "4096", "runtimePlatform": { "cpuArchitecture": "X86_64", "operatingSystemFamily": "LINUX" From 8439c60e8a48d43f6bbfc694ea37b83b28b424d0 Mon Sep 17 00:00:00 2001 From: John Clary Date: Thu, 30 Oct 2025 13:35:23 -0400 Subject: [PATCH 35/37] john testing: revert everything --- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index 1b4c32ca35..13f6433e54 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -37,8 +37,8 @@ "awslogs-stream-prefix": "ecs" } }, - "memory": 4096, - "memoryReservation": 4096, + "memory": 2048, + "memoryReservation": 2048, "mountPoints": [ ], "name": "atd-moped", "portMappings": [ @@ -87,7 +87,7 @@ "FARGATE" ], "cpu": "256", - "memory": "4096", + "memory": "2048", "runtimePlatform": { "cpuArchitecture": "X86_64", "operatingSystemFamily": "LINUX" From 93dc1d758d9da73cdf85cca877c3fd718177810b Mon Sep 17 00:00:00 2001 From: John Clary Date: Thu, 30 Oct 2025 13:41:40 -0400 Subject: [PATCH 36/37] john testing: add env var --- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index 13f6433e54..b746cd03d6 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -15,6 +15,10 @@ { "name": "HASURA_GRAPHQL_ENABLE_CONSOLE", "value": "false" + }, + { + "name": "FAKE_ENV", + "value": "hello" } ], "essential": true, From ebe3748b359f1ceb25c3adeb2bc78370485410f2 Mon Sep 17 00:00:00 2001 From: John Clary Date: Thu, 30 Oct 2025 13:45:37 -0400 Subject: [PATCH 37/37] john testing: remove env var --- .../ecs_task_definitions/staging.graphql-engine.ecs-td.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json index b746cd03d6..13f6433e54 100644 --- a/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json +++ b/moped-database/ecs_task_definitions/staging.graphql-engine.ecs-td.json @@ -15,10 +15,6 @@ { "name": "HASURA_GRAPHQL_ENABLE_CONSOLE", "value": "false" - }, - { - "name": "FAKE_ENV", - "value": "hello" } ], "essential": true,