Skip to content

fix: prefer user region then all other region #125

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 11, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 45 additions & 8 deletions scripts/validate_model_deployment_quota.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,19 @@ param (
[string]$ModelsParameter
)

# Verify all required parameters are provided
# Read from environment variables (do not pass in azure.yaml)
$AiServiceName = $env:AZURE_AISERVICE_NAME
$ResourceGroup = $env:AZURE_RESOURCE_GROUP

# Validate required parameters
$MissingParams = @()

if (-not $SubscriptionId) {
$MissingParams += "subscription"
}

if (-not $Location) {
$MissingParams += "location"
}

if (-not $ModelsParameter) {
$MissingParams += "models-parameter"
}
Expand All @@ -25,20 +27,56 @@ if ($MissingParams.Count -gt 0) {
exit 1
}

# Load main.parameters.json
$JsonContent = Get-Content -Path "./infra/main.parameters.json" -Raw | ConvertFrom-Json

if (-not $JsonContent) {
Write-Error "❌ ERROR: Failed to parse main.parameters.json. Ensure the JSON file is valid."
exit 1
}

$aiModelDeployments = $JsonContent.parameters.$ModelsParameter.value

if (-not $aiModelDeployments -or -not ($aiModelDeployments -is [System.Collections.IEnumerable])) {
Write-Error "❌ ERROR: The specified property $ModelsParameter does not exist or is not an array."
Write-Error "❌ ERROR: The specified property '$ModelsParameter' does not exist or is not an array."
exit 1
}

# Check if AI resource + all deployments already exist
if ($AiServiceName -and $ResourceGroup) {
$existing = az cognitiveservices account show `
--name $AiServiceName `
--resource-group $ResourceGroup `
--query "name" --output tsv 2>$null

if ($existing) {
$deployedModels = az cognitiveservices account deployment list `
--name $AiServiceName `
--resource-group $ResourceGroup `
--query "[].name" --output tsv 2>$null

$requiredDeployments = @()
foreach ($deployment in $aiModelDeployments) {
$requiredDeployments += $deployment.name
}

$missingDeployments = @()
foreach ($required in $requiredDeployments) {
if ($deployedModels -notcontains $required) {
$missingDeployments += $required
}
}

if ($missingDeployments.Count -eq 0) {
Write-Host "ℹ️ Azure AI service '$AiServiceName' exists and all required model deployments are provisioned."
Write-Host "⏭️ Skipping quota validation."
exit 0
} else {
Write-Host "🔍 AI service exists, but the following model deployments are missing: $($missingDeployments -join ', ')"
Write-Host "➡️ Proceeding with quota validation for missing models..."
}
}
}

# Start quota validation
az account set --subscription $SubscriptionId
Write-Host "🎯 Active Subscription: $(az account show --query '[name, id]' --output tsv)"

Expand All @@ -56,8 +94,7 @@ foreach ($deployment in $aiModelDeployments) {

if ($exitCode -ne 0) {
if ($exitCode -eq 2) {
# Quota error already printed inside the script, exit gracefully without reprinting
exit 1
exit 1 # already printed, graceful
}
Write-Error "❌ ERROR: Quota validation failed for model deployment: $name"
$QuotaAvailable = $false
Expand Down
98 changes: 63 additions & 35 deletions scripts/validate_model_deployment_quota.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,37 +19,66 @@ while [[ $# -gt 0 ]]; do
shift 2
;;
*)
echo "Unknown option: $1"
echo "❌ ERROR: Unknown option: $1"
exit 1
;;
esac
done

# Verify all required parameters are provided and echo missing ones
# Validate required parameters
MISSING_PARAMS=()
[[ -z "$SUBSCRIPTION_ID" ]] && MISSING_PARAMS+=("subscription")
[[ -z "$LOCATION" ]] && MISSING_PARAMS+=("location")
[[ -z "$MODELS_PARAMETER" ]] && MISSING_PARAMS+=("models-parameter")

if [[ -z "$SUBSCRIPTION_ID" ]]; then
MISSING_PARAMS+=("subscription")
if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then
echo "❌ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}"
echo "Usage: $0 --subscription <SUBSCRIPTION_ID> --location <LOCATION> --models-parameter <MODELS_PARAMETER>"
exit 1
fi

if [[ -z "$LOCATION" ]]; then
MISSING_PARAMS+=("location")
fi
# Read from environment
AISERVICE_NAME="${AZURE_AISERVICE_NAME}"
RESOURCE_GROUP="${AZURE_RESOURCE_GROUP}"

if [[ -z "$MODELS_PARAMETER" ]]; then
MISSING_PARAMS+=("models-parameter")
fi
# Check service and deployment existence
if [[ -n "$AISERVICE_NAME" && -n "$RESOURCE_GROUP" ]]; then
existing=$(az cognitiveservices account show --name "$AISERVICE_NAME" --resource-group "$RESOURCE_GROUP" --query "name" --output tsv 2>/dev/null)
if [[ -n "$existing" ]]; then
echo "ℹ️ Found Azure AI service: $AISERVICE_NAME"

if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then
echo "❌ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}"
echo "Usage: $0 --subscription <SUBSCRIPTION_ID> --location <LOCATION> --models-parameter <MODELS_PARAMETER>"
exit 1
existing_deployments=$(az cognitiveservices account deployment list --name "$AISERVICE_NAME" --resource-group "$RESOURCE_GROUP" --query "[].name" --output tsv 2>/dev/null)

# Extract required model names
required_models=$(jq -r ".parameters.$MODELS_PARAMETER.value[].name" ./infra/main.parameters.json 2>/dev/null)

if [[ -z "$required_models" ]]; then
echo "❌ ERROR: Failed to extract required model names from main.parameters.json"
exit 1
fi

all_present=true
for model in $required_models; do
if ! grep -q -w "$model" <<< "$existing_deployments"; then
all_present=false
break
fi
done

if [[ "$all_present" == "true" ]]; then
echo "✅ All required model deployments already exist in AI service '$AISERVICE_NAME'."
echo "⏭️ Skipping quota validation."
exit 0
else
echo "🔍 AI service exists but some model deployments are missing — proceeding with quota validation."
fi
fi
fi

# If we reach here, continue with normal quota checks
aiModelDeployments=$(jq -c ".parameters.$MODELS_PARAMETER.value[]" ./infra/main.parameters.json)

if [ $? -ne 0 ]; then
echo "Error: Failed to parse main.parameters.json. Ensure jq is installed and the JSON file is valid."
if [[ $? -ne 0 ]]; then
echo "❌ ERROR: Failed to parse main.parameters.json. Ensure jq is installed and the JSON is valid."
exit 1
fi

Expand All @@ -59,30 +88,29 @@ echo "🎯 Active Subscription: $(az account show --query '[name, id]' --output
quotaAvailable=true

while IFS= read -r deployment; do
name=${AZURE_ENV_MODEL_NAME:-$(echo "$deployment" | jq -r '.name')}
model=${AZURE_ENV_MODEL_NAME:-$(echo "$deployment" | jq -r '.model.name')}
type=${AZURE_ENV_MODEL_DEPLOYMENT_TYPE:-$(echo "$deployment" | jq -r '.sku.name')}
capacity=${AZURE_ENV_MODEL_CAPACITY:-$(echo "$deployment" | jq -r '.sku.capacity')}
name=${AZURE_ENV_MODEL_NAME:-$(echo "$deployment" | jq -r '.name')}
model=${AZURE_ENV_MODEL_NAME:-$(echo "$deployment" | jq -r '.model.name')}
type=${AZURE_ENV_MODEL_DEPLOYMENT_TYPE:-$(echo "$deployment" | jq -r '.sku.name')}
capacity=${AZURE_ENV_MODEL_CAPACITY:-$(echo "$deployment" | jq -r '.sku.capacity')}

echo "🔍 Validating model deployment: $name ..."
./scripts/validate_model_quota.sh --location "$LOCATION" --model "$model" --capacity $capacity --deployment-type $type
./scripts/validate_model_quota.sh --location "$LOCATION" --model "$model" --capacity "$capacity" --deployment-type "$type"

# Check if the script failed
exit_code=$?
if [ $exit_code -ne 0 ]; then
if [ $exit_code -eq 2 ]; then
# Skip printing any quota validation error — already handled inside the validation script
exit 1
fi
echo "❌ ERROR: Quota validation failed for model deployment: $name"
quotaAvailable=false
if [[ $exit_code -ne 0 ]]; then
if [[ $exit_code -eq 2 ]]; then
# Quota validation handled inside script — stop immediately
exit 1
fi
echo "❌ ERROR: Quota validation failed for model deployment: $name"
quotaAvailable=false
fi
done <<< "$(echo "$aiModelDeployments")"

if [ "$quotaAvailable" = false ]; then
echo "❌ ERROR: One or more model deployments failed validation."
exit 1
if [[ "$quotaAvailable" = false ]]; then
echo "❌ ERROR: One or more model deployments failed validation."
exit 1
else
echo "✅ All model deployments passed quota validation successfully."
exit 0
echo "✅ All model deployments passed quota validation successfully."
exit 0
fi
85 changes: 47 additions & 38 deletions scripts/validate_model_quota.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ param (
[int]$Capacity
)

# Verify required parameters
# Validate parameters
$MissingParams = @()
if (-not $Location) { $MissingParams += "location" }
if (-not $Model) { $MissingParams += "model" }
Expand All @@ -24,7 +24,6 @@ if ($DeploymentType -ne "Standard" -and $DeploymentType -ne "GlobalStandard") {
}

$ModelType = "OpenAI.$DeploymentType.$Model"

$PreferredRegions = @('australiaeast', 'eastus', 'eastus2', 'francecentral', 'japaneast', 'norwayeast', 'southindia', 'swedencentral', 'uksouth', 'westus', 'westus3')
$AllResults = @()

Expand All @@ -33,66 +32,76 @@ function Check-Quota {
[string]$Region
)

$ModelInfoRaw = az cognitiveservices usage list --location $Region --query "[?name.value=='$ModelType']" --output json
$ModelInfo = $null

try {
$ModelInfoRaw = az cognitiveservices usage list --location $Region --query "[?name.value=='$ModelType']" --output json
$ModelInfo = $ModelInfoRaw | ConvertFrom-Json
if (-not $ModelInfo) { return }

$CurrentValue = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).currentValue
$Limit = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).limit

$CurrentValue = [int]($CurrentValue -replace '\.0+$', '')
$Limit = [int]($Limit -replace '\.0+$', '')
$Available = $Limit - $CurrentValue

return [PSCustomObject]@{
Region = $Region
Model = $ModelType
Limit = $Limit
Used = $CurrentValue
Available = $Available
}
} catch {
return
}
}

if (-not $ModelInfo) {
return
# First, check the user-specified region
Write-Host "`n🔍 Checking quota in the requested region '$Location'..."
$PrimaryResult = Check-Quota -Region $Location

if ($PrimaryResult) {
$AllResults += $PrimaryResult
if ($PrimaryResult.Available -ge $Capacity) {
Write-Host "`n✅ Sufficient quota found in original region '$Location'."
exit 0
} else {
Write-Host "`n⚠️ Insufficient quota in '$Location' (Available: $($PrimaryResult.Available), Required: $Capacity). Checking fallback regions..."
}
} else {
Write-Host "`n⚠️ Could not retrieve quota info for region '$Location'. Checking fallback regions..."
}

$CurrentValue = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).currentValue
$Limit = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).limit

$CurrentValue = [int]($CurrentValue -replace '\.0+$', '')
$Limit = [int]($Limit -replace '\.0+$', '')
$Available = $Limit - $CurrentValue
# Remove primary region from fallback list
$FallbackRegions = $PreferredRegions | Where-Object { $_ -ne $Location }

$script:AllResults += [PSCustomObject]@{
Region = $Region
Model = $ModelType
Limit = $Limit
Used = $CurrentValue
Available = $Available
foreach ($region in $FallbackRegions) {
$result = Check-Quota -Region $region
if ($result) {
$AllResults += $result
}
}

foreach ($region in $PreferredRegions) {
Check-Quota -Region $region
}

# Display Results Table
Write-Host "\n-------------------------------------------------------------------------------------------------------------"
Write-Host "`n-------------------------------------------------------------------------------------------------------------"
Write-Host "| No. | Region | Model Name | Limit | Used | Available |"
Write-Host "-------------------------------------------------------------------------------------------------------------"

$count = 1
foreach ($entry in $AllResults) {
$index = $PreferredRegions.IndexOf($entry.Region) + 1
$modelShort = $entry.Model.Substring($entry.Model.LastIndexOf(".") + 1)
Write-Host ("| {0,-4} | {1,-16} | {2,-35} | {3,-7} | {4,-7} | {5,-9} |" -f $index, $entry.Region, $entry.Model, $entry.Limit, $entry.Used, $entry.Available)
Write-Host ("| {0,-4} | {1,-16} | {2,-35} | {3,-7} | {4,-7} | {5,-9} |" -f $count, $entry.Region, $entry.Model, $entry.Limit, $entry.Used, $entry.Available)
$count++
}
Write-Host "-------------------------------------------------------------------------------------------------------------"

$EligibleRegion = $AllResults | Where-Object { $_.Region -eq $Location -and $_.Available -ge $Capacity }
if ($EligibleRegion) {
Write-Host "\n✅ Sufficient quota found in original region '$Location'."
exit 0
}
# Suggest fallback regions
$EligibleFallbacks = $AllResults | Where-Object { $_.Region -ne $Location -and $_.Available -ge $Capacity }

$FallbackRegions = $AllResults | Where-Object { $_.Region -ne $Location -and $_.Available -ge $Capacity }

if ($FallbackRegions.Count -gt 0) {
Write-Host "`n❌ Deployment cannot proceed because the original region '$Location' lacks sufficient quota."
if ($EligibleFallbacks.Count -gt 0) {
Write-Host "`n❌ Deployment cannot proceed in '$Location'."
Write-Host "➡️ You can retry using one of the following regions with sufficient quota:`n"

foreach ($region in $FallbackRegions) {
foreach ($region in $EligibleFallbacks) {
Write-Host " • $($region.Region) (Available: $($region.Available))"
}

Expand All @@ -104,5 +113,5 @@ if ($FallbackRegions.Count -gt 0) {
exit 2
}

Write-Error "❌ ERROR: No available quota found in any region."
Write-Error "`n❌ ERROR: No available quota found in any region."
exit 1
Loading