Rollback ws, on staging (2 revisions) #84
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rollback Deployment | |
| run-name: > | |
| Rollback | |
| ${{ | |
| github.event.inputs.rollback_api == 'true' && 'api, ' || '' | |
| }}${{ | |
| github.event.inputs.rollback_worker == 'true' && 'worker, ' || '' | |
| }}${{ | |
| github.event.inputs.rollback_ws == 'true' && 'ws, ' || '' | |
| }}${{ | |
| github.event.inputs.rollback_webhook == 'true' && 'webhook ' || '' | |
| }}on ${{ github.event.inputs.environment }} (${{ | |
| github.event.inputs.revisions_to_rollback | |
| }} revisions) | |
| description: Rollback deployment to the previous task definition for selected services in the specified environment. | |
| concurrency: | |
| group: "rollback-${{ github.event.inputs.environment }}" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'Environment to rollback' | |
| required: true | |
| type: choice | |
| default: staging | |
| options: | |
| - staging | |
| - production-us | |
| - production-eu | |
| - production-both | |
| rollback_api: | |
| description: 'Rollback API' | |
| required: true | |
| type: boolean | |
| default: true | |
| rollback_worker: | |
| description: 'Rollback Worker' | |
| required: true | |
| type: boolean | |
| default: true | |
| rollback_ws: | |
| description: 'Rollback WS' | |
| required: true | |
| type: boolean | |
| default: true | |
| rollback_webhook: | |
| description: 'Rollback Webhook' | |
| required: true | |
| type: boolean | |
| default: true | |
| revisions_to_rollback: | |
| description: 'Number of revisions to rollback (default: 1)' | |
| required: true | |
| type: number | |
| default: 1 | |
| rollback_signoff: | |
| description: "This will rollback the selected services to the previous task definition. This won't rollback any database migration or environment changes. Do you agree?" | |
| required: true | |
| type: choice | |
| default: 'I do not agree' | |
| options: | |
| - 'I agree' | |
| - 'I do not agree' | |
| jobs: | |
| prepare-matrix: | |
| runs-on: ubuntu-latest | |
| if : "${{ github.event.inputs.rollback_signoff == 'I agree' }}" | |
| outputs: | |
| env_matrix: ${{ steps.set-matrix.outputs.env_matrix }} | |
| service_matrix: ${{ steps.set-matrix.outputs.service_matrix }} | |
| rollback_matrix: ${{ steps.set-matrix.outputs.rollback_matrix }} | |
| steps: | |
| - name: Validate Selected Services | |
| run: | | |
| if [ "${{ github.event.inputs.rollback_api }}" != "true" ] && \ | |
| [ "${{ github.event.inputs.rollback_worker }}" != "true" ] && \ | |
| [ "${{ github.event.inputs.rollback_ws }}" != "true" ] && \ | |
| [ "${{ github.event.inputs.rollback_webhook }}" != "true" ]; then | |
| echo "Error: At least one service must be selected for rollback." | |
| exit 1 | |
| fi | |
| - name: Generate Environment, Service, and Rollback Matrices | |
| id: set-matrix | |
| env: | |
| WORKER_SERVICE: ${{ vars.WORKER_SERVICE }} | |
| run: | | |
| envs=() | |
| services=() | |
| rollback_matrix=() | |
| # Collect selected environments | |
| if [ "${{ github.event.inputs.environment }}" == "staging" ]; then | |
| envs+=("\"staging-eu\"") | |
| fi | |
| if [ "${{ github.event.inputs.environment }}" == "production-us" ]; then | |
| envs+=("\"prod-us\"") | |
| fi | |
| if [ "${{ github.event.inputs.environment }}" == "production-eu" ]; then | |
| envs+=("\"prod-eu\"") | |
| fi | |
| if [ "${{ github.event.inputs.environment }}" == "production-both" ]; then | |
| envs+=("\"prod-us\"") | |
| envs+=("\"prod-eu\"") | |
| fi | |
| # Collect selected services | |
| if [ "${{ github.event.inputs.rollback_api }}" == "true" ]; then | |
| services+=("\"api\"") | |
| fi | |
| if [ "${{ github.event.inputs.rollback_worker }}" == "true" ]; then | |
| services+=("\"worker\"") | |
| fi | |
| if [ "${{ github.event.inputs.rollback_ws }}" == "true" ]; then | |
| services+=("\"ws\"") | |
| fi | |
| if [ "${{ github.event.inputs.rollback_webhook }}" == "true" ]; then | |
| services+=("\"webhook\"") | |
| fi | |
| # Parse service secrets and generate rollback_matrix | |
| for service in "${services[@]}"; do | |
| if [ "$service" == "\"worker\"" ]; then | |
| IFS=',' read -r -a worker_services <<< "$WORKER_SERVICE" | |
| for worker_service in $(echo "$WORKER_SERVICE" | jq -c '.[]'); do | |
| cluster_name=$(echo "$worker_service" | jq -r '.cluster_name') | |
| container_name=$(echo "$worker_service" | jq -r '.container_name') | |
| service_name=$(echo "$worker_service" | jq -r '.service') | |
| task_name=$(echo "$worker_service" | jq -r '.task_name') | |
| image=$(echo "$worker_service" | jq -r '.image') | |
| rollback_matrix+=("{\"cluster_name\": \"$cluster_name\", \"container_name\": \"$container_name\", \"service_name\": \"$service_name\", \"task_name\": \"$task_name\", \"image\": \"$image\"}") | |
| done | |
| elif [ "$service" == "\"api\"" ]; then | |
| cluster_name=api-cluster | |
| container_name=api-container | |
| service_name=api-service | |
| task_name=api-task | |
| image=api | |
| rollback_matrix+=("{\"cluster_name\": \"$cluster_name\", \"container_name\": \"$container_name\", \"service_name\": \"$service_name\", \"task_name\": \"$task_name\", \"image\": \"$image\"}") | |
| elif [ "$service" == "\"ws\"" ]; then | |
| cluster_name=ws-cluster | |
| container_name=ws-container | |
| service_name=ws-service | |
| task_name=ws-task | |
| image=ws | |
| rollback_matrix+=("{\"cluster_name\": \"$cluster_name\", \"container_name\": \"$container_name\", \"service_name\": \"$service_name\", \"task_name\": \"$task_name\", \"image\": \"$image\"}") | |
| elif [ "$service" == "\"webhook\"" ]; then | |
| cluster_name=webhook-cluster | |
| container_name=webhook-container | |
| service_name=webhook-service | |
| task_name=webhook-task | |
| image=webhook | |
| rollback_matrix+=("{\"cluster_name\": \"$cluster_name\", \"container_name\": \"$container_name\", \"service_name\": \"$service_name\", \"task_name\": \"$task_name\", \"image\": \"$image\"}") | |
| fi | |
| done | |
| env_matrix="{\"environment\": [$( | |
| IFS=','; echo "${envs[*]}" | |
| )]}" | |
| service_matrix="{\"service\": [$( | |
| IFS=','; echo "${services[*]}" | |
| )]}" | |
| rollback_matrix="[$( | |
| IFS=','; echo "${rollback_matrix[*]}" | |
| )]" | |
| echo "env_matrix=$env_matrix" >> $GITHUB_OUTPUT | |
| echo "service_matrix=$service_matrix" >> $GITHUB_OUTPUT | |
| echo "rollback_matrix=$rollback_matrix" >> $GITHUB_OUTPUT | |
| rollback: | |
| needs: [prepare-matrix] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| env: ${{ fromJson(needs.prepare-matrix.outputs.env_matrix).environment }} | |
| service: ${{ fromJson(needs.prepare-matrix.outputs.rollback_matrix) }} | |
| environment: ${{ matrix.env }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| - name: ECS get output | |
| env: | |
| TASK_NAME: ${{ vars.ECS_PREFIX }}-${{ matrix.service.task_name }} | |
| CONTAINER_NAME: ${{ vars.ECS_PREFIX }}-${{ matrix.service.container_name }} | |
| SERVICE_NAME: ${{ vars.ECS_PREFIX }}-${{ matrix.service.service_name }} | |
| CLUSTER_NAME: ${{ vars.ECS_PREFIX }}-${{ matrix.service.cluster_name }} | |
| id: ecs-output | |
| run: | | |
| echo "Retrieving current_task_definition_arn..." | |
| current_task_definition_arn=$(aws ecs describe-services --cluster ${CLUSTER_NAME} --services ${SERVICE_NAME} --query 'services[0].taskDefinition' --output text) | |
| echo "current_task_definition_arn=$current_task_definition_arn" >> $GITHUB_ENV | |
| echo "Retrieving task_definition_family..." | |
| task_definition_family=$(aws ecs describe-task-definition --task-definition ${TASK_NAME} --query 'taskDefinition.family' --output text) | |
| echo "task_definition_family=$task_definition_family" >> $GITHUB_ENV | |
| echo "Retrieving task_definition_list..." | |
| task_definition_list=$(aws ecs list-task-definitions --family-prefix "${task_definition_family}" --output text --sort DESC | grep 'TASKDEFINITIONARNS' | cut -f 2) | |
| task_definition_list_formatted=$(echo "$task_definition_list" | tr '\n' '|') # Replace newline with '|' | |
| echo "task_definition_list=$task_definition_list_formatted" >> $GITHUB_ENV | |
| if [ -n "$task_definition_list" ]; then | |
| echo "Retrieving previous_task_definition_arn..." | |
| index=$(echo "$task_definition_list" | grep -n "$current_task_definition_arn" | cut -d ':' -f 1) | |
| if [ -n "$index" ]; then | |
| if [ "$index" -ge 1 ]; then # Greater than or equal to 1 | |
| revisions_to_rollback=${{ github.event.inputs.revisions_to_rollback }} | |
| previous_index=$((index + revisions_to_rollback)) | |
| previous_task_definition_arn=$(echo "$task_definition_list" | sed -n "${previous_index}p") | |
| if [ -z "$previous_task_definition_arn" ]; then | |
| echo "Error: Cannot rollback $revisions_to_rollback revisions. Not enough previous versions available." | |
| exit 1 | |
| fi | |
| echo "previous_task_definition_arn=$previous_task_definition_arn" >> $GITHUB_ENV | |
| else | |
| echo "Invalid index value: $index" | |
| fi | |
| else | |
| echo "Previous task definition not found. It seems to me someone deleted the current task from the list and that is why I can't find the previous task." | |
| exit 1 | |
| fi | |
| else | |
| echo "No task definitions found." | |
| exit 1 | |
| fi | |
| - name: Rollback a service to the previous task definition | |
| id: rollback-service | |
| env: | |
| PREVIOUS_TASK: ${{ env.previous_task_definition_arn }} | |
| CURRENT_TASK: ${{ env.current_task_definition_arn }} | |
| SERVICE_NAME: ${{ vars.ECS_PREFIX }}-${{ matrix.service.service_name }} | |
| CLUSTER_NAME: ${{ vars.ECS_PREFIX }}-${{ matrix.service.cluster_name }} | |
| run: | | |
| aws ecs update-service --cluster ${CLUSTER_NAME} --service ${SERVICE_NAME} --task-definition ${{ env.PREVIOUS_TASK }} | |
| aws ecs wait services-stable --cluster ${CLUSTER_NAME} --service ${SERVICE_NAME} | |
| echo "After Rollback:" | |
| echo "The previous task definition: $(echo $CURRENT_TASK | awk -F'task-definition/' '{print $2}')" | |
| echo "The current task definition: $(echo $PREVIOUS_TASK | awk -F'task-definition/' '{print $2}')" | |
| echo "Rollback completed successfully." | |