From 56913ab677c7e9af49fa853b093bf8a91e46e7f2 Mon Sep 17 00:00:00 2001 From: "Priyanka Singhal (Persistent Systems Inc)" Date: Mon, 24 Feb 2025 18:00:40 +0530 Subject: [PATCH] Add quota check in CI pipeline to verify availability in supported region and prevent failures due to insufficient quota. --- .github/workflows/ci.yml | 48 +++++++++++++++++++- scripts/checkquota.sh | 97 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 scripts/checkquota.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54ff3a089..791e653bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,52 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + # Run Quota Check Script + - name: Run Quota Check + id: quota-check + run: | + export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }} + export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }} + export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }} + export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" + export GPT_MIN_CAPACITY="130" + export TEXT_EMBEDDING_MIN_CAPACITY="30" + + chmod +x scripts/checkquota.sh + if ! scripts/checkquota.sh; then + # If quota check fails due to insufficient quota, set the flag + if grep -q "No region with sufficient quota found" scripts/checkquota.sh; then + echo "QUOTA_FAILED=true" >> $GITHUB_ENV + fi + exit 1 # Fail the pipeline if any other failure occurs + fi + + - name: Send Notification on Quota Failure + if: env.QUOTA_FAILED == 'true' + run: | + RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + EMAIL_BODY=$(cat <Dear Team,

The quota check has failed, and the pipeline cannot proceed.

Build URL: ${RUN_URL}

Please take necessary action.

Best regards,
Your Automation Team

" + } + EOF + ) + + curl -X POST "${{ secrets.LOGIC_APP_URL }}" \ + -H "Content-Type: application/json" \ + -d "$EMAIL_BODY" || echo "Failed to send notification" + + - name: Fail Pipeline if Quota Check Fails + if: env.QUOTA_FAILED == 'true' + run: exit 1 + + # The pipeline stops here if quota check fails! + + - name: Set Deployment Region + run: | + echo "Selected Region: $VALID_REGION" + echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -51,7 +97,7 @@ jobs: uses: devcontainers/ci@v0.3 env: AZURE_ENV_NAME: ${{ github.run_id }} - AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} + AZURE_LOCATION: ${{ env.AZURE_LOCATION }} with: imageName: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator cacheFrom: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator diff --git a/scripts/checkquota.sh b/scripts/checkquota.sh new file mode 100644 index 000000000..ddcb4dddf --- /dev/null +++ b/scripts/checkquota.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# List of Azure regions to check for quota (update as needed) +REGIONS=("eastus" "westus" "northcentralus" "uksouth" "swedencentral") + +SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID}" +GPT_MIN_CAPACITY="${GPT_MIN_CAPACITY}" +TEXT_EMBEDDING_MIN_CAPACITY="${TEXT_EMBEDDING_MIN_CAPACITY}" +AZURE_CLIENT_ID="${AZURE_CLIENT_ID}" +AZURE_TENANT_ID="${AZURE_TENANT_ID}" +AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET}" + +# Authenticate using Managed Identity +echo "Authentication using Managed Identity..." +if ! az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID"; then + echo "❌ Error: Failed to login using Managed Identity." + exit 1 +fi + +echo "🔄 Validating required environment variables..." +if [[ -z "$SUBSCRIPTION_ID" || -z "$GPT_MIN_CAPACITY" || -z "$TEXT_EMBEDDING_MIN_CAPACITY" ]]; then + echo "❌ ERROR: Missing required environment variables." + exit 1 +fi + +echo "🔄 Setting Azure subscription..." +if ! az account set --subscription "$SUBSCRIPTION_ID"; then + echo "❌ ERROR: Invalid subscription ID or insufficient permissions." + exit 1 +fi +echo "✅ Azure subscription set successfully." + +# Define models and their minimum required capacities +declare -A MIN_CAPACITY=( + ["OpenAI.Standard.gpt-4o"]=$GPT_MIN_CAPACITY + ["OpenAI.Standard.text-embedding-ada-002"]=$TEXT_EMBEDDING_MIN_CAPACITY +) + +VALID_REGION="" +for REGION in "${REGIONS[@]}"; do + echo "----------------------------------------" + echo "🔍 Checking region: $REGION" + + QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json) + if [ -z "$QUOTA_INFO" ]; then + echo "⚠️ WARNING: Failed to retrieve quota for region $REGION. Skipping." + continue + fi + + INSUFFICIENT_QUOTA=false + for MODEL in "${!MIN_CAPACITY[@]}"; do + MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL\"" ' + BEGIN { RS="},"; FS="," } + $0 ~ model { print $0 } + ') + + if [ -z "$MODEL_INFO" ]; then + echo "⚠️ WARNING: No quota information found for model: $MODEL in $REGION. Skipping." + continue + fi + + CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentValue"/ {print $2}' | tr -d ',' | tr -d ' ') + LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ') + + CURRENT_VALUE=${CURRENT_VALUE:-0} + LIMIT=${LIMIT:-0} + + CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1) + LIMIT=$(echo "$LIMIT" | cut -d'.' -f1) + + AVAILABLE=$((LIMIT - CURRENT_VALUE)) + + echo "✅ Model: $MODEL | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE" + + if [ "$AVAILABLE" -lt "${MIN_CAPACITY[$MODEL]}" ]; then + echo "❌ ERROR: $MODEL in $REGION has insufficient quota." + INSUFFICIENT_QUOTA=true + break + fi + done + + if [ "$INSUFFICIENT_QUOTA" = false ]; then + VALID_REGION="$REGION" + break + fi + +done + +if [ -z "$VALID_REGION" ]; then + echo "❌ No region with sufficient quota found. Blocking deployment." + echo "QUOTA_FAILED=true" >> "$GITHUB_ENV" + exit 0 +else + echo "✅ Final Region: $VALID_REGION" + echo "VALID_REGION=$VALID_REGION" >> "$GITHUB_ENV" + exit 0 +fi