From 2ee0beb9f6a1d8d5f3da23c6313f6712db8e6527 Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Thu, 28 Nov 2024 20:10:29 +0530 Subject: [PATCH 01/13] Added Role based assignment to connect to db for web app --- infra/app/adminweb.bicep | 2 + infra/app/web.bicep | 1 + .../database/deploy_create_table_script.bicep | 4 +- infra/main.bicep | 2 + infra/main.json | 50 ++++++++++++++++--- .../data_scripts/create_postgres_tables.py | 38 ++++++++++++++ scripts/run_create_table_script.sh | 4 ++ 7 files changed, 92 insertions(+), 9 deletions(-) diff --git a/infra/app/adminweb.bicep b/infra/app/adminweb.bicep index 56397ed6f..82f4611cc 100644 --- a/infra/app/adminweb.bicep +++ b/infra/app/adminweb.bicep @@ -28,6 +28,7 @@ param speechKeyName string = '' param authType string param dockerFullImageName string = '' param useDocker bool = dockerFullImageName != '' +param databaseType string = 'cosmos' // 'cosmos' or 'postgres' var azureFormRecognizerInfoUpdated = useKeyVault ? azureFormRecognizerInfo @@ -68,6 +69,7 @@ module adminweb '../core/host/appservice.bicep' = { scmDoBuildDuringDeployment: useDocker ? false : true applicationInsightsName: applicationInsightsName appServicePlanId: appServicePlanId + managedIdentity: databaseType == 'postgres' appSettings: union(appSettings, { AZURE_AUTH_TYPE: authType USE_KEY_VAULT: useKeyVault ? useKeyVault : '' diff --git a/infra/app/web.bicep b/infra/app/web.bicep index 90e6cb1f3..4fb24d3c0 100644 --- a/infra/app/web.bicep +++ b/infra/app/web.bicep @@ -155,6 +155,7 @@ module web '../core/host/appservice.bicep' = { dockerFullImageName: dockerFullImageName scmDoBuildDuringDeployment: useDocker ? false : true healthCheckPath: healthCheckPath + managedIdentity: databaseType == 'postgres' } } diff --git a/infra/core/database/deploy_create_table_script.bicep b/infra/core/database/deploy_create_table_script.bicep index b1fc012df..afccc8cc8 100644 --- a/infra/core/database/deploy_create_table_script.bicep +++ b/infra/core/database/deploy_create_table_script.bicep @@ -5,6 +5,8 @@ param baseUrl string param keyVaultName string param identity string param postgresSqlServerName string +param webAppPrincipalName string +param adminAppPrincipalName string resource create_index 'Microsoft.Resources/deploymentScripts@2020-10-01' = { kind:'AzureCLI' @@ -19,7 +21,7 @@ resource create_index 'Microsoft.Resources/deploymentScripts@2020-10-01' = { properties: { azCliVersion: '2.52.0' primaryScriptUri: '${baseUrl}scripts/run_create_table_script.sh' - arguments: '${baseUrl} ${keyVaultName} ${resourceGroup().name} ${postgresSqlServerName}' // Specify any arguments for the script + arguments: '${baseUrl} ${keyVaultName} ${resourceGroup().name} ${postgresSqlServerName} ${webAppPrincipalName} ${adminAppPrincipalName}' // Specify any arguments for the script timeout: 'PT1H' // Specify the desired timeout duration retentionInterval: 'PT1H' // Specify the desired retention interval cleanupPreference:'OnSuccess' diff --git a/infra/main.bicep b/infra/main.bicep index f4696a2a9..1a3901656 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -1238,6 +1238,8 @@ module createIndex './core/database/deploy_create_table_script.bicep' = if (dat baseUrl:baseUrl keyVaultName:keyvault.outputs.name postgresSqlServerName: postgresDBModule.outputs.postgresDbOutput.postgresSQLName + webAppPrincipalName: web_docker.outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID + adminAppPrincipalName: adminweb_docker.outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID } scope: rg dependsOn:[keyvault, postgresDBModule, storekeys] diff --git a/infra/main.json b/infra/main.json index 314c1f8d3..dd98b37ff 100644 --- a/infra/main.json +++ b/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "16387401552448615008" + "templateHash": "9521961093814280778" } }, "parameters": { @@ -2671,7 +2671,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "64857797928052295" + "templateHash": "8216675081949619536" } }, "parameters": { @@ -2852,6 +2852,9 @@ "scmDoBuildDuringDeployment": "[if(parameters('useDocker'), createObject('value', false()), createObject('value', true()))]", "healthCheckPath": { "value": "[parameters('healthCheckPath')]" + }, + "managedIdentity": { + "value": "[equals(parameters('databaseType'), 'postgres')]" } }, "template": { @@ -3667,7 +3670,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "64857797928052295" + "templateHash": "8216675081949619536" } }, "parameters": { @@ -3848,6 +3851,9 @@ "scmDoBuildDuringDeployment": "[if(parameters('useDocker'), createObject('value', false()), createObject('value', true()))]", "healthCheckPath": { "value": "[parameters('healthCheckPath')]" + }, + "managedIdentity": { + "value": "[equals(parameters('databaseType'), 'postgres')]" } }, "template": { @@ -4700,7 +4706,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "12567732396765618168" + "templateHash": "14757216181799932354" } }, "parameters": { @@ -4815,6 +4821,10 @@ "useDocker": { "type": "bool", "defaultValue": "[not(equals(parameters('dockerFullImageName'), ''))]" + }, + "databaseType": { + "type": "string", + "defaultValue": "cosmos" } }, "resources": [ @@ -4860,6 +4870,9 @@ "appServicePlanId": { "value": "[parameters('appServicePlanId')]" }, + "managedIdentity": { + "value": "[equals(parameters('databaseType'), 'postgres')]" + }, "appSettings": { "value": "[union(parameters('appSettings'), createObject('AZURE_AUTH_TYPE', parameters('authType'), 'USE_KEY_VAULT', if(parameters('useKeyVault'), parameters('useKeyVault'), ''), 'AZURE_OPENAI_API_KEY', if(parameters('useKeyVault'), parameters('openAIKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('azureOpenAIName')), '2023-05-01').key1), 'AZURE_SEARCH_KEY', if(parameters('useKeyVault'), parameters('searchKeyName'), listAdminKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Search/searchServices', parameters('azureAISearchName')), '2021-04-01-preview').primaryKey), 'AZURE_BLOB_STORAGE_INFO', if(parameters('useKeyVault'), parameters('azureBlobStorageInfo'), replace(parameters('azureBlobStorageInfo'), '$STORAGE_ACCOUNT_KEY', listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-09-01').keys[0].value)), 'AZURE_FORM_RECOGNIZER_INFO', if(parameters('useKeyVault'), parameters('azureFormRecognizerInfo'), replace(parameters('azureFormRecognizerInfo'), '$FORM_RECOGNIZER_KEY', listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('formRecognizerName')), '2023-05-01').key1)), 'AZURE_CONTENT_SAFETY_KEY', if(parameters('useKeyVault'), parameters('contentSafetyKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('contentSafetyName')), '2023-05-01').key1), 'AZURE_SPEECH_SERVICE_KEY', if(parameters('useKeyVault'), parameters('speechKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('speechServiceName')), '2023-05-01').key1), 'AZURE_COMPUTER_VISION_KEY', if(or(parameters('useKeyVault'), equals(parameters('computerVisionName'), '')), parameters('computerVisionKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('computerVisionName')), '2023-05-01').key1)))]" } @@ -5648,7 +5661,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "12567732396765618168" + "templateHash": "14757216181799932354" } }, "parameters": { @@ -5763,6 +5776,10 @@ "useDocker": { "type": "bool", "defaultValue": "[not(equals(parameters('dockerFullImageName'), ''))]" + }, + "databaseType": { + "type": "string", + "defaultValue": "cosmos" } }, "resources": [ @@ -5808,6 +5825,9 @@ "appServicePlanId": { "value": "[parameters('appServicePlanId')]" }, + "managedIdentity": { + "value": "[equals(parameters('databaseType'), 'postgres')]" + }, "appSettings": { "value": "[union(parameters('appSettings'), createObject('AZURE_AUTH_TYPE', parameters('authType'), 'USE_KEY_VAULT', if(parameters('useKeyVault'), parameters('useKeyVault'), ''), 'AZURE_OPENAI_API_KEY', if(parameters('useKeyVault'), parameters('openAIKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('azureOpenAIName')), '2023-05-01').key1), 'AZURE_SEARCH_KEY', if(parameters('useKeyVault'), parameters('searchKeyName'), listAdminKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Search/searchServices', parameters('azureAISearchName')), '2021-04-01-preview').primaryKey), 'AZURE_BLOB_STORAGE_INFO', if(parameters('useKeyVault'), parameters('azureBlobStorageInfo'), replace(parameters('azureBlobStorageInfo'), '$STORAGE_ACCOUNT_KEY', listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-09-01').keys[0].value)), 'AZURE_FORM_RECOGNIZER_INFO', if(parameters('useKeyVault'), parameters('azureFormRecognizerInfo'), replace(parameters('azureFormRecognizerInfo'), '$FORM_RECOGNIZER_KEY', listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('formRecognizerName')), '2023-05-01').key1)), 'AZURE_CONTENT_SAFETY_KEY', if(parameters('useKeyVault'), parameters('contentSafetyKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('contentSafetyName')), '2023-05-01').key1), 'AZURE_SPEECH_SERVICE_KEY', if(parameters('useKeyVault'), parameters('speechKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('speechServiceName')), '2023-05-01').key1), 'AZURE_COMPUTER_VISION_KEY', if(or(parameters('useKeyVault'), equals(parameters('computerVisionName'), '')), parameters('computerVisionKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('computerVisionName')), '2023-05-01').key1)))]" } @@ -11999,6 +12019,12 @@ }, "postgresSqlServerName": { "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'deploy_postgres_sql'), '2022-09-01').outputs.postgresDbOutput.value.postgresSQLName]" + }, + "webAppPrincipalName": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('websiteName'))), '2022-09-01').outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID.value]" + }, + "adminAppPrincipalName": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('adminWebsiteName'))), '2022-09-01').outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID.value]" } }, "template": { @@ -12008,7 +12034,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "754015545513025215" + "templateHash": "12297908402052822068" } }, "parameters": { @@ -12029,6 +12055,12 @@ }, "postgresSqlServerName": { "type": "string" + }, + "webAppPrincipalName": { + "type": "string" + }, + "adminAppPrincipalName": { + "type": "string" } }, "resources": [ @@ -12047,7 +12079,7 @@ "properties": { "azCliVersion": "2.52.0", "primaryScriptUri": "[format('{0}scripts/run_create_table_script.sh', parameters('baseUrl'))]", - "arguments": "[format('{0} {1} {2} {3}', parameters('baseUrl'), parameters('keyVaultName'), resourceGroup().name, parameters('postgresSqlServerName'))]", + "arguments": "[format('{0} {1} {2} {3} {4} {5}', parameters('baseUrl'), parameters('keyVaultName'), resourceGroup().name, parameters('postgresSqlServerName'), parameters('webAppPrincipalName'), parameters('adminAppPrincipalName'))]", "timeout": "PT1H", "retentionInterval": "PT1H", "cleanupPreference": "OnSuccess" @@ -12057,11 +12089,13 @@ } }, "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('adminWebsiteName')))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'keyvault')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'deploy_managed_identity')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'deploy_postgres_sql')]", "[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('rgName'))]", - "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys')]" + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys')]", + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('websiteName')))]" ] } ], diff --git a/scripts/data_scripts/create_postgres_tables.py b/scripts/data_scripts/create_postgres_tables.py index 92df9fa93..2f125644a 100644 --- a/scripts/data_scripts/create_postgres_tables.py +++ b/scripts/data_scripts/create_postgres_tables.py @@ -2,8 +2,11 @@ from azure.keyvault.secrets import SecretClient from azure.identity import DefaultAzureCredential import psycopg2 +from psycopg2 import sql key_vault_name = "kv_to-be-replaced" +principal_name = "webAppPrincipalName" +admin_principal_name = "adminAppPrincipalName" def get_secrets_from_kv(kv_name, secret_name): credential = DefaultAzureCredential() @@ -13,6 +16,37 @@ def get_secrets_from_kv(kv_name, secret_name): return secret_client.get_secret(secret_name).value +def grant_permissions(cursor, dbname, schema_name, principal_name): + """ + Grants database and schema-level permissions to a specified principal. + + Parameters: + - cursor: psycopg2 cursor object for database operations. + - dbname: Name of the database to grant CONNECT permission. + - schema_name: Name of the schema to grant table-level permissions. + - principal_name: Name of the principal (role or user) to grant permissions. + """ + # Grant CONNECT on database + grant_connect_query = sql.SQL("GRANT CONNECT ON DATABASE {database} TO {principal}") + cursor.execute( + grant_connect_query.format( + database=sql.Identifier(dbname), + principal=sql.Identifier(principal_name), + ) + ) + print(f"Granted CONNECT on database '{dbname}' to '{principal_name}'") + + # Grant SELECT, INSERT, UPDATE, DELETE on schema tables + grant_permissions_query = sql.SQL( + "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA {schema} TO {principal}" + ) + cursor.execute( + grant_permissions_query.format( + schema=sql.Identifier(schema_name), + principal=sql.Identifier(principal_name), + ) + ) + postgres_details = json.loads(get_secrets_from_kv(key_vault_name, "AZURE-POSTGRESQL-INFO")) host = postgres_details.get("host", "") user = postgres_details.get("user", "") @@ -30,6 +64,10 @@ def get_secrets_from_kv(kv_name, secret_name): conn = psycopg2.connect(conn_string) cursor = conn.cursor() +grant_permissions(cursor, dbname, "public", principal_name) +grant_permissions(cursor, dbname, "public", admin_principal_name) +conn.commit() + # Drop and recreate the conversations table cursor.execute("DROP TABLE IF EXISTS conversations") conn.commit() diff --git a/scripts/run_create_table_script.sh b/scripts/run_create_table_script.sh index 1d3bd2063..6ed535644 100644 --- a/scripts/run_create_table_script.sh +++ b/scripts/run_create_table_script.sh @@ -8,6 +8,8 @@ requirementFile="requirements.txt" requirementFileUrl=${baseUrl}"scripts/data_scripts/requirements.txt" resourceGroup="$3" serverName="$4" +webAppPrincipalName = "$5" +adminAppPrincipalName = "$6" echo "Script Started" @@ -27,6 +29,8 @@ echo "Download completed" #Replace key vault name sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "create_postgres_tables.py" +sed -i "s/webAppPrincipalName/${webAppPrincipalName}/g" "create_postgres_tables.py" +sed -i "s/adminAppPrincipalName/${adminAppPrincipalName}/g" "create_postgres_tables.py" pip install -r requirements.txt From 4e9d5ab83a2dc90d73040430ff0a2808d8b94b47 Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Thu, 28 Nov 2024 20:15:33 +0530 Subject: [PATCH 02/13] Added dependencies --- infra/main.bicep | 8 +++++--- infra/main.json | 12 +++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/infra/main.bicep b/infra/main.bicep index 1a3901656..9f4817b1e 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -1238,11 +1238,13 @@ module createIndex './core/database/deploy_create_table_script.bicep' = if (dat baseUrl:baseUrl keyVaultName:keyvault.outputs.name postgresSqlServerName: postgresDBModule.outputs.postgresDbOutput.postgresSQLName - webAppPrincipalName: web_docker.outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID - adminAppPrincipalName: adminweb_docker.outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID + webAppPrincipalName: hostingModel == 'code' ? web.outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID : web_docker.outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID + adminAppPrincipalName: hostingModel == 'code' ? adminweb.outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID : adminweb_docker.outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID } scope: rg - dependsOn:[keyvault, postgresDBModule, storekeys] + dependsOn: hostingModel == 'code' ? [keyvault, postgresDBModule, storekeys, web, adminweb] : [ + [keyvault, postgresDBModule, storekeys, web_docker, adminweb_docker] + ] } output APPLICATIONINSIGHTS_CONNECTION_STRING string = monitoring.outputs.applicationInsightsConnectionString diff --git a/infra/main.json b/infra/main.json index dd98b37ff..e1ed4febf 100644 --- a/infra/main.json +++ b/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "9521961093814280778" + "templateHash": "831473305954833848" } }, "parameters": { @@ -12020,12 +12020,8 @@ "postgresSqlServerName": { "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'deploy_postgres_sql'), '2022-09-01').outputs.postgresDbOutput.value.postgresSQLName]" }, - "webAppPrincipalName": { - "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('websiteName'))), '2022-09-01').outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID.value]" - }, - "adminAppPrincipalName": { - "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('adminWebsiteName'))), '2022-09-01').outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID.value]" - } + "webAppPrincipalName": "[if(equals(parameters('hostingModel'), 'code'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('websiteName')), '2022-09-01').outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID.value), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('websiteName'))), '2022-09-01').outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID.value))]", + "adminAppPrincipalName": "[if(equals(parameters('hostingModel'), 'code'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('adminWebsiteName')), '2022-09-01').outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID.value), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('adminWebsiteName'))), '2022-09-01').outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID.value))]" }, "template": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", @@ -12089,12 +12085,14 @@ } }, "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('adminWebsiteName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('adminWebsiteName')))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'keyvault')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'deploy_managed_identity')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'deploy_postgres_sql')]", "[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('rgName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys')]", + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('websiteName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('websiteName')))]" ] } From e54f13ecbc28a0d8f5f45f12f655281a6350fb81 Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Thu, 28 Nov 2024 22:41:49 +0530 Subject: [PATCH 03/13] Modified the script --- scripts/run_create_table_script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run_create_table_script.sh b/scripts/run_create_table_script.sh index 6ed535644..7583013ea 100644 --- a/scripts/run_create_table_script.sh +++ b/scripts/run_create_table_script.sh @@ -8,8 +8,8 @@ requirementFile="requirements.txt" requirementFileUrl=${baseUrl}"scripts/data_scripts/requirements.txt" resourceGroup="$3" serverName="$4" -webAppPrincipalName = "$5" -adminAppPrincipalName = "$6" +webAppPrincipalName="$5" +adminAppPrincipalName="$6" echo "Script Started" From 0b841c1d911a1b1dd052e609d1d522fdde8cbbdc Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Thu, 28 Nov 2024 22:51:56 +0530 Subject: [PATCH 04/13] Fix: Added Principal to DB --- scripts/data_scripts/create_postgres_tables.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/data_scripts/create_postgres_tables.py b/scripts/data_scripts/create_postgres_tables.py index 2f125644a..434692d66 100644 --- a/scripts/data_scripts/create_postgres_tables.py +++ b/scripts/data_scripts/create_postgres_tables.py @@ -26,6 +26,14 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): - schema_name: Name of the schema to grant table-level permissions. - principal_name: Name of the principal (role or user) to grant permissions. """ + + add_principal_user_query = "SELECT * FROM pgaadauth_create_principal({principal}, false, false)" + cursor.execute( + add_principal_user_query.format( + principal=sql.Identifier(principal_name), + ) + ) + # Grant CONNECT on database grant_connect_query = sql.SQL("GRANT CONNECT ON DATABASE {database} TO {principal}") cursor.execute( From 9079910462195045e15563d310e49d5aec7c9d96 Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Thu, 28 Nov 2024 23:01:11 +0530 Subject: [PATCH 05/13] Fix: Added Managed identity to DB --- infra/app/web.bicep | 2 +- infra/core/database/postgresdb.bicep | 2 +- infra/main.bicep | 6 +++-- infra/main.json | 22 +++++++++++++------ .../data_scripts/create_postgres_tables.py | 2 +- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/infra/app/web.bicep b/infra/app/web.bicep index 4fb24d3c0..755f6eb32 100644 --- a/infra/app/web.bicep +++ b/infra/app/web.bicep @@ -211,7 +211,7 @@ resource cosmosRoleDefinition 'Microsoft.DocumentDB/databaseAccounts/sqlRoleDefi name: '${json(appSettings.AZURE_COSMOSDB_INFO).accountName}/00000000-0000-0000-0000-000000000002' } -module cosmosUserRole '../core/database/cosmos-sql-role-assign.bicep' = { +module cosmosUserRole '../core/database/cosmos-sql-role-assign.bicep' = if(databaseType == 'cosmos') { name: 'cosmos-sql-user-role-${web.name}' params: { accountName: json(appSettings.AZURE_COSMOSDB_INFO).accountName diff --git a/infra/core/database/postgresdb.bicep b/infra/core/database/postgresdb.bicep index 32a2af714..557732cd8 100644 --- a/infra/core/database/postgresdb.bicep +++ b/infra/core/database/postgresdb.bicep @@ -65,7 +65,7 @@ resource delayScript 'Microsoft.Resources/deploymentScripts@2020-10-01' = { kind: 'AzurePowerShell' properties: { azPowerShellVersion: '3.0' - scriptContent: 'start-sleep -Seconds 180' + scriptContent: 'start-sleep -Seconds 300' cleanupPreference: 'Always' retentionInterval: 'PT1H' } diff --git a/infra/main.bicep b/infra/main.bicep index 9f4817b1e..d71d08f85 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -813,6 +813,7 @@ module adminweb './app/adminweb.bicep' = if (hostingModel == 'code') { useKeyVault: useKeyVault keyVaultName: useKeyVault || authType == 'rbac' ? keyvault.outputs.name : '' authType: authType + databaseType: databaseType appSettings: { AZURE_COMPUTER_VISION_ENDPOINT: useAdvancedImageProcessing ? computerVision.outputs.endpoint : '' AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION: computerVisionVectorizeImageApiVersion @@ -886,6 +887,7 @@ module adminweb_docker './app/adminweb.bicep' = if (hostingModel == 'container') useKeyVault: useKeyVault keyVaultName: useKeyVault || authType == 'rbac' ? keyvault.outputs.name : '' authType: authType + databaseType: databaseType appSettings: { AZURE_COMPUTER_VISION_ENDPOINT: useAdvancedImageProcessing ? computerVision.outputs.endpoint : '' AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION: computerVisionVectorizeImageApiVersion @@ -1238,8 +1240,8 @@ module createIndex './core/database/deploy_create_table_script.bicep' = if (dat baseUrl:baseUrl keyVaultName:keyvault.outputs.name postgresSqlServerName: postgresDBModule.outputs.postgresDbOutput.postgresSQLName - webAppPrincipalName: hostingModel == 'code' ? web.outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID : web_docker.outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID - adminAppPrincipalName: hostingModel == 'code' ? adminweb.outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID : adminweb_docker.outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID + webAppPrincipalName: hostingModel == 'code' ? web.outputs.FRONTEND_API_NAME : web_docker.outputs.FRONTEND_API_NAME + adminAppPrincipalName: hostingModel == 'code' ? adminweb.outputs.WEBSITE_ADMIN_NAME : adminweb_docker.outputs.WEBSITE_ADMIN_NAME } scope: rg dependsOn: hostingModel == 'code' ? [keyvault, postgresDBModule, storekeys, web, adminweb] : [ diff --git a/infra/main.json b/infra/main.json index e1ed4febf..c88fc267c 100644 --- a/infra/main.json +++ b/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "831473305954833848" + "templateHash": "8458218652856054228" } }, "parameters": { @@ -977,7 +977,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "7878676541226628436" + "templateHash": "17455459966134467610" } }, "parameters": { @@ -1091,7 +1091,7 @@ "kind": "AzurePowerShell", "properties": { "azPowerShellVersion": "3.0", - "scriptContent": "start-sleep -Seconds 180", + "scriptContent": "start-sleep -Seconds 300", "cleanupPreference": "Always", "retentionInterval": "PT1H" }, @@ -2671,7 +2671,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "8216675081949619536" + "templateHash": "8696900365541524252" } }, "parameters": { @@ -3490,6 +3490,7 @@ ] }, { + "condition": "[equals(parameters('databaseType'), 'cosmos')]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", "name": "[format('cosmos-sql-user-role-{0}', format('{0}-app-module', parameters('name')))]", @@ -3670,7 +3671,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "8216675081949619536" + "templateHash": "8696900365541524252" } }, "parameters": { @@ -4489,6 +4490,7 @@ ] }, { + "condition": "[equals(parameters('databaseType'), 'cosmos')]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", "name": "[format('cosmos-sql-user-role-{0}', format('{0}-app-module', parameters('name')))]", @@ -4653,6 +4655,9 @@ "authType": { "value": "[parameters('authType')]" }, + "databaseType": { + "value": "[parameters('databaseType')]" + }, "appSettings": { "value": { "AZURE_COMPUTER_VISION_ENDPOINT": "[if(parameters('useAdvancedImageProcessing'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.endpoint.value, '')]", @@ -5608,6 +5613,9 @@ "authType": { "value": "[parameters('authType')]" }, + "databaseType": { + "value": "[parameters('databaseType')]" + }, "appSettings": { "value": { "AZURE_COMPUTER_VISION_ENDPOINT": "[if(parameters('useAdvancedImageProcessing'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.endpoint.value, '')]", @@ -12020,8 +12028,8 @@ "postgresSqlServerName": { "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'deploy_postgres_sql'), '2022-09-01').outputs.postgresDbOutput.value.postgresSQLName]" }, - "webAppPrincipalName": "[if(equals(parameters('hostingModel'), 'code'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('websiteName')), '2022-09-01').outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID.value), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('websiteName'))), '2022-09-01').outputs.FRONTEND_API_IDENTITY_PRINCIPAL_ID.value))]", - "adminAppPrincipalName": "[if(equals(parameters('hostingModel'), 'code'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('adminWebsiteName')), '2022-09-01').outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID.value), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('adminWebsiteName'))), '2022-09-01').outputs.WEBSITE_ADMIN_IDENTITY_PRINCIPAL_ID.value))]" + "webAppPrincipalName": "[if(equals(parameters('hostingModel'), 'code'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('websiteName')), '2022-09-01').outputs.FRONTEND_API_NAME.value), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('websiteName'))), '2022-09-01').outputs.FRONTEND_API_NAME.value))]", + "adminAppPrincipalName": "[if(equals(parameters('hostingModel'), 'code'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('adminWebsiteName')), '2022-09-01').outputs.WEBSITE_ADMIN_NAME.value), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('adminWebsiteName'))), '2022-09-01').outputs.WEBSITE_ADMIN_NAME.value))]" }, "template": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", diff --git a/scripts/data_scripts/create_postgres_tables.py b/scripts/data_scripts/create_postgres_tables.py index 434692d66..d5a57c284 100644 --- a/scripts/data_scripts/create_postgres_tables.py +++ b/scripts/data_scripts/create_postgres_tables.py @@ -27,7 +27,7 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): - principal_name: Name of the principal (role or user) to grant permissions. """ - add_principal_user_query = "SELECT * FROM pgaadauth_create_principal({principal}, false, false)" + add_principal_user_query = sql.SQL("SELECT * FROM pgaadauth_create_principal({principal}, false, false)") cursor.execute( add_principal_user_query.format( principal=sql.Identifier(principal_name), From 07ef813b5c8f78bf2eb4a141a69ab4c10c3c1da7 Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Thu, 28 Nov 2024 23:24:31 +0530 Subject: [PATCH 06/13] fix: Modified the paramater --- scripts/run_create_table_script.sh | 2 - scripts/test/create_postgres_tables.py | 140 +++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 scripts/test/create_postgres_tables.py diff --git a/scripts/run_create_table_script.sh b/scripts/run_create_table_script.sh index 7583013ea..eeef0934e 100644 --- a/scripts/run_create_table_script.sh +++ b/scripts/run_create_table_script.sh @@ -34,6 +34,4 @@ sed -i "s/adminAppPrincipalName/${adminAppPrincipalName}/g" "create_postgres_tab pip install -r requirements.txt -pip show azure-identity - python create_postgres_tables.py diff --git a/scripts/test/create_postgres_tables.py b/scripts/test/create_postgres_tables.py new file mode 100644 index 000000000..187966522 --- /dev/null +++ b/scripts/test/create_postgres_tables.py @@ -0,0 +1,140 @@ +import json +from azure.keyvault.secrets import SecretClient +from azure.identity import DefaultAzureCredential +import psycopg2 +from psycopg2 import sql + +key_vault_name = "kv-wpvykucviclze" +principal_name = "web-wpvykucviclze-docker" +admin_principal_name = "web-wpvykucviclze-admin-docker" + +def get_secrets_from_kv(kv_name, secret_name): + credential = DefaultAzureCredential() + secret_client = SecretClient( + vault_url=f"https://{key_vault_name}.vault.azure.net/", credential=credential + ) # Create a secret client object using the credential and Key Vault name + return secret_client.get_secret(secret_name).value + + +def grant_permissions(cursor, dbname, schema_name, principal_name): + """ + Grants database and schema-level permissions to a specified principal. + + Parameters: + - cursor: psycopg2 cursor object for database operations. + - dbname: Name of the database to grant CONNECT permission. + - schema_name: Name of the schema to grant table-level permissions. + - principal_name: Name of the principal (role or user) to grant permissions. + """ + + add_principal_user_query = sql.SQL("SELECT * FROM pgaadauth_create_principal({principal}, false, false)") + cursor.execute( + add_principal_user_query.format( + principal=sql.Literal(principal_name), + ) + ) + + # Grant CONNECT on database + grant_connect_query = sql.SQL("GRANT CONNECT ON DATABASE {database} TO {principal}") + cursor.execute( + grant_connect_query.format( + database=sql.Identifier(dbname), + principal=sql.Identifier(principal_name), + ) + ) + print(f"Granted CONNECT on database '{dbname}' to '{principal_name}'") + + # Grant SELECT, INSERT, UPDATE, DELETE on schema tables + grant_permissions_query = sql.SQL( + "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA {schema} TO {principal}" + ) + cursor.execute( + grant_permissions_query.format( + schema=sql.Identifier(schema_name), + principal=sql.Identifier(principal_name), + ) + ) + +postgres_details = json.loads(get_secrets_from_kv(key_vault_name, "AZURE-POSTGRESQL-INFO")) +host = postgres_details.get("host", "") +user = postgres_details.get("user", "") +dbname = postgres_details.get("dbname", "") +password = postgres_details.get("password", "") + +# Acquire the access token +cred = DefaultAzureCredential() +access_token = cred.get_token("https://ossrdbms-aad.database.windows.net/.default") + +# Combine the token with the connection string to establish the connection. +conn_string = "host={0} user={1} dbname={2} password={3}".format( + host, user, dbname, password +) +conn = psycopg2.connect(conn_string) +cursor = conn.cursor() + +grant_permissions(cursor, dbname, "public", principal_name) +grant_permissions(cursor, dbname, "public", admin_principal_name) +conn.commit() + +# Drop and recreate the conversations table +cursor.execute("DROP TABLE IF EXISTS conversations") +conn.commit() + +create_cs_sql = """CREATE TABLE conversations ( + id TEXT PRIMARY KEY, + conversation_id TEXT NOT NULL, + type TEXT NOT NULL, + "createdAt" TEXT, + "updatedAt" TEXT, + user_id TEXT NOT NULL, + title TEXT + );""" +cursor.execute(create_cs_sql) +conn.commit() + +# Drop and recreate the messages table +cursor.execute("DROP TABLE IF EXISTS messages") +conn.commit() + +create_ms_sql = """CREATE TABLE messages ( + id TEXT PRIMARY KEY, + type VARCHAR(50) NOT NULL, + "createdAt" TEXT, + "updatedAt" TEXT, + user_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + role VARCHAR(50), + content TEXT NOT NULL, + feedback TEXT + );""" +cursor.execute(create_ms_sql) +conn.commit() + +# Add pg_diskann extension and search_indexes table +cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_diskann CASCADE;") +conn.commit() + +cursor.execute("DROP TABLE IF EXISTS search_indexes;") +conn.commit() + +table_create_command = """CREATE TABLE IF NOT EXISTS search_indexes( + id text, + title text, + chunk integer, + chunk_id text, + "offset" integer, + page_number integer, + content text, + source text, + metadata text, + content_vector public.vector(1536) +);""" + +cursor.execute(table_create_command) +conn.commit() + +cursor.execute("CREATE INDEX search_indexes_content_vector_diskann_idx ON search_indexes USING diskann (content_vector vector_cosine_ops);") +conn.commit() + +cursor.close() +conn.close() From 110d666a9d87cc671c930eea1ad0b41f7aa4e351 Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Thu, 28 Nov 2024 23:30:45 +0530 Subject: [PATCH 07/13] fix: parameter changes --- .../data_scripts/create_postgres_tables.py | 2 +- scripts/test/create_postgres_tables.py | 140 ------------------ 2 files changed, 1 insertion(+), 141 deletions(-) delete mode 100644 scripts/test/create_postgres_tables.py diff --git a/scripts/data_scripts/create_postgres_tables.py b/scripts/data_scripts/create_postgres_tables.py index d5a57c284..55dbf719a 100644 --- a/scripts/data_scripts/create_postgres_tables.py +++ b/scripts/data_scripts/create_postgres_tables.py @@ -30,7 +30,7 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): add_principal_user_query = sql.SQL("SELECT * FROM pgaadauth_create_principal({principal}, false, false)") cursor.execute( add_principal_user_query.format( - principal=sql.Identifier(principal_name), + principal=sql.Literal(principal_name), ) ) diff --git a/scripts/test/create_postgres_tables.py b/scripts/test/create_postgres_tables.py deleted file mode 100644 index 187966522..000000000 --- a/scripts/test/create_postgres_tables.py +++ /dev/null @@ -1,140 +0,0 @@ -import json -from azure.keyvault.secrets import SecretClient -from azure.identity import DefaultAzureCredential -import psycopg2 -from psycopg2 import sql - -key_vault_name = "kv-wpvykucviclze" -principal_name = "web-wpvykucviclze-docker" -admin_principal_name = "web-wpvykucviclze-admin-docker" - -def get_secrets_from_kv(kv_name, secret_name): - credential = DefaultAzureCredential() - secret_client = SecretClient( - vault_url=f"https://{key_vault_name}.vault.azure.net/", credential=credential - ) # Create a secret client object using the credential and Key Vault name - return secret_client.get_secret(secret_name).value - - -def grant_permissions(cursor, dbname, schema_name, principal_name): - """ - Grants database and schema-level permissions to a specified principal. - - Parameters: - - cursor: psycopg2 cursor object for database operations. - - dbname: Name of the database to grant CONNECT permission. - - schema_name: Name of the schema to grant table-level permissions. - - principal_name: Name of the principal (role or user) to grant permissions. - """ - - add_principal_user_query = sql.SQL("SELECT * FROM pgaadauth_create_principal({principal}, false, false)") - cursor.execute( - add_principal_user_query.format( - principal=sql.Literal(principal_name), - ) - ) - - # Grant CONNECT on database - grant_connect_query = sql.SQL("GRANT CONNECT ON DATABASE {database} TO {principal}") - cursor.execute( - grant_connect_query.format( - database=sql.Identifier(dbname), - principal=sql.Identifier(principal_name), - ) - ) - print(f"Granted CONNECT on database '{dbname}' to '{principal_name}'") - - # Grant SELECT, INSERT, UPDATE, DELETE on schema tables - grant_permissions_query = sql.SQL( - "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA {schema} TO {principal}" - ) - cursor.execute( - grant_permissions_query.format( - schema=sql.Identifier(schema_name), - principal=sql.Identifier(principal_name), - ) - ) - -postgres_details = json.loads(get_secrets_from_kv(key_vault_name, "AZURE-POSTGRESQL-INFO")) -host = postgres_details.get("host", "") -user = postgres_details.get("user", "") -dbname = postgres_details.get("dbname", "") -password = postgres_details.get("password", "") - -# Acquire the access token -cred = DefaultAzureCredential() -access_token = cred.get_token("https://ossrdbms-aad.database.windows.net/.default") - -# Combine the token with the connection string to establish the connection. -conn_string = "host={0} user={1} dbname={2} password={3}".format( - host, user, dbname, password -) -conn = psycopg2.connect(conn_string) -cursor = conn.cursor() - -grant_permissions(cursor, dbname, "public", principal_name) -grant_permissions(cursor, dbname, "public", admin_principal_name) -conn.commit() - -# Drop and recreate the conversations table -cursor.execute("DROP TABLE IF EXISTS conversations") -conn.commit() - -create_cs_sql = """CREATE TABLE conversations ( - id TEXT PRIMARY KEY, - conversation_id TEXT NOT NULL, - type TEXT NOT NULL, - "createdAt" TEXT, - "updatedAt" TEXT, - user_id TEXT NOT NULL, - title TEXT - );""" -cursor.execute(create_cs_sql) -conn.commit() - -# Drop and recreate the messages table -cursor.execute("DROP TABLE IF EXISTS messages") -conn.commit() - -create_ms_sql = """CREATE TABLE messages ( - id TEXT PRIMARY KEY, - type VARCHAR(50) NOT NULL, - "createdAt" TEXT, - "updatedAt" TEXT, - user_id TEXT NOT NULL, - conversation_id TEXT NOT NULL, - role VARCHAR(50), - content TEXT NOT NULL, - feedback TEXT - );""" -cursor.execute(create_ms_sql) -conn.commit() - -# Add pg_diskann extension and search_indexes table -cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_diskann CASCADE;") -conn.commit() - -cursor.execute("DROP TABLE IF EXISTS search_indexes;") -conn.commit() - -table_create_command = """CREATE TABLE IF NOT EXISTS search_indexes( - id text, - title text, - chunk integer, - chunk_id text, - "offset" integer, - page_number integer, - content text, - source text, - metadata text, - content_vector public.vector(1536) -);""" - -cursor.execute(table_create_command) -conn.commit() - -cursor.execute("CREATE INDEX search_indexes_content_vector_diskann_idx ON search_indexes USING diskann (content_vector vector_cosine_ops);") -conn.commit() - -cursor.close() -conn.close() From d697cd66c2943b3408de1a71e469cda5563ec2ff Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Thu, 28 Nov 2024 23:37:31 +0530 Subject: [PATCH 08/13] test: for connectivity with aad --- scripts/data_scripts/create_postgres_tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/data_scripts/create_postgres_tables.py b/scripts/data_scripts/create_postgres_tables.py index 55dbf719a..893b9bd63 100644 --- a/scripts/data_scripts/create_postgres_tables.py +++ b/scripts/data_scripts/create_postgres_tables.py @@ -57,7 +57,7 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): postgres_details = json.loads(get_secrets_from_kv(key_vault_name, "AZURE-POSTGRESQL-INFO")) host = postgres_details.get("host", "") -user = postgres_details.get("user", "") +user = "wpvykucviclze-managed-identity" dbname = postgres_details.get("dbname", "") password = postgres_details.get("password", "") @@ -67,7 +67,7 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): # Combine the token with the connection string to establish the connection. conn_string = "host={0} user={1} dbname={2} password={3}".format( - host, user, dbname, password + host, user, dbname, access_token ) conn = psycopg2.connect(conn_string) cursor = conn.cursor() From 5ab585b77d8dde3759aa53f424dc413c2d6581ff Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Thu, 28 Nov 2024 23:42:26 +0530 Subject: [PATCH 09/13] fix: access token issue --- scripts/data_scripts/create_postgres_tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/data_scripts/create_postgres_tables.py b/scripts/data_scripts/create_postgres_tables.py index 893b9bd63..c2d3d0927 100644 --- a/scripts/data_scripts/create_postgres_tables.py +++ b/scripts/data_scripts/create_postgres_tables.py @@ -67,7 +67,7 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): # Combine the token with the connection string to establish the connection. conn_string = "host={0} user={1} dbname={2} password={3}".format( - host, user, dbname, access_token + host, user, dbname, access_token.token ) conn = psycopg2.connect(conn_string) cursor = conn.cursor() From cd74dfdc6210719951ac63f8fb03ce9e5326c160 Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Thu, 28 Nov 2024 23:50:54 +0530 Subject: [PATCH 10/13] Updated script --- scripts/data_scripts/create_postgres_tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/data_scripts/create_postgres_tables.py b/scripts/data_scripts/create_postgres_tables.py index c2d3d0927..460edc9a6 100644 --- a/scripts/data_scripts/create_postgres_tables.py +++ b/scripts/data_scripts/create_postgres_tables.py @@ -66,7 +66,7 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): access_token = cred.get_token("https://ossrdbms-aad.database.windows.net/.default") # Combine the token with the connection string to establish the connection. -conn_string = "host={0} user={1} dbname={2} password={3}".format( +conn_string = "host={0} user={1} dbname={2} password={3} sslmode=require".format( host, user, dbname, access_token.token ) conn = psycopg2.connect(conn_string) From 5f29e5bb737693d2c8c222497ecb760a8c44f159 Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Fri, 29 Nov 2024 00:05:22 +0530 Subject: [PATCH 11/13] fix: Updates script to run with managed identity --- .../core/database/deploy_create_table_script.bicep | 3 ++- infra/main.bicep | 1 + infra/main.json | 14 ++++++++++---- scripts/data_scripts/create_postgres_tables.py | 2 +- scripts/run_create_table_script.sh | 2 ++ 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/infra/core/database/deploy_create_table_script.bicep b/infra/core/database/deploy_create_table_script.bicep index afccc8cc8..4b2718af2 100644 --- a/infra/core/database/deploy_create_table_script.bicep +++ b/infra/core/database/deploy_create_table_script.bicep @@ -7,6 +7,7 @@ param identity string param postgresSqlServerName string param webAppPrincipalName string param adminAppPrincipalName string +param managedIdentityName string resource create_index 'Microsoft.Resources/deploymentScripts@2020-10-01' = { kind:'AzureCLI' @@ -21,7 +22,7 @@ resource create_index 'Microsoft.Resources/deploymentScripts@2020-10-01' = { properties: { azCliVersion: '2.52.0' primaryScriptUri: '${baseUrl}scripts/run_create_table_script.sh' - arguments: '${baseUrl} ${keyVaultName} ${resourceGroup().name} ${postgresSqlServerName} ${webAppPrincipalName} ${adminAppPrincipalName}' // Specify any arguments for the script + arguments: '${baseUrl} ${keyVaultName} ${resourceGroup().name} ${postgresSqlServerName} ${webAppPrincipalName} ${adminAppPrincipalName} ${managedIdentityName}' // Specify any arguments for the script timeout: 'PT1H' // Specify the desired timeout duration retentionInterval: 'PT1H' // Specify the desired retention interval cleanupPreference:'OnSuccess' diff --git a/infra/main.bicep b/infra/main.bicep index d71d08f85..d0c237d3f 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -1242,6 +1242,7 @@ module createIndex './core/database/deploy_create_table_script.bicep' = if (dat postgresSqlServerName: postgresDBModule.outputs.postgresDbOutput.postgresSQLName webAppPrincipalName: hostingModel == 'code' ? web.outputs.FRONTEND_API_NAME : web_docker.outputs.FRONTEND_API_NAME adminAppPrincipalName: hostingModel == 'code' ? adminweb.outputs.WEBSITE_ADMIN_NAME : adminweb_docker.outputs.WEBSITE_ADMIN_NAME + managedIdentityName: managedIdentityModule.outputs.managedIdentityOutput.name } scope: rg dependsOn: hostingModel == 'code' ? [keyvault, postgresDBModule, storekeys, web, adminweb] : [ diff --git a/infra/main.json b/infra/main.json index c88fc267c..07e114325 100644 --- a/infra/main.json +++ b/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "8458218652856054228" + "templateHash": "936545603138294666" } }, "parameters": { @@ -12029,7 +12029,10 @@ "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'deploy_postgres_sql'), '2022-09-01').outputs.postgresDbOutput.value.postgresSQLName]" }, "webAppPrincipalName": "[if(equals(parameters('hostingModel'), 'code'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('websiteName')), '2022-09-01').outputs.FRONTEND_API_NAME.value), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('websiteName'))), '2022-09-01').outputs.FRONTEND_API_NAME.value))]", - "adminAppPrincipalName": "[if(equals(parameters('hostingModel'), 'code'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('adminWebsiteName')), '2022-09-01').outputs.WEBSITE_ADMIN_NAME.value), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('adminWebsiteName'))), '2022-09-01').outputs.WEBSITE_ADMIN_NAME.value))]" + "adminAppPrincipalName": "[if(equals(parameters('hostingModel'), 'code'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('adminWebsiteName')), '2022-09-01').outputs.WEBSITE_ADMIN_NAME.value), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('adminWebsiteName'))), '2022-09-01').outputs.WEBSITE_ADMIN_NAME.value))]", + "managedIdentityName": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'deploy_managed_identity'), '2022-09-01').outputs.managedIdentityOutput.value.name]" + } }, "template": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", @@ -12038,7 +12041,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "12297908402052822068" + "templateHash": "5666855677780801837" } }, "parameters": { @@ -12065,6 +12068,9 @@ }, "adminAppPrincipalName": { "type": "string" + }, + "managedIdentityName": { + "type": "string" } }, "resources": [ @@ -12083,7 +12089,7 @@ "properties": { "azCliVersion": "2.52.0", "primaryScriptUri": "[format('{0}scripts/run_create_table_script.sh', parameters('baseUrl'))]", - "arguments": "[format('{0} {1} {2} {3} {4} {5}', parameters('baseUrl'), parameters('keyVaultName'), resourceGroup().name, parameters('postgresSqlServerName'), parameters('webAppPrincipalName'), parameters('adminAppPrincipalName'))]", + "arguments": "[format('{0} {1} {2} {3} {4} {5} {6}', parameters('baseUrl'), parameters('keyVaultName'), resourceGroup().name, parameters('postgresSqlServerName'), parameters('webAppPrincipalName'), parameters('adminAppPrincipalName'), parameters('managedIdentityName'))]", "timeout": "PT1H", "retentionInterval": "PT1H", "cleanupPreference": "OnSuccess" diff --git a/scripts/data_scripts/create_postgres_tables.py b/scripts/data_scripts/create_postgres_tables.py index 460edc9a6..247cb30ca 100644 --- a/scripts/data_scripts/create_postgres_tables.py +++ b/scripts/data_scripts/create_postgres_tables.py @@ -7,6 +7,7 @@ key_vault_name = "kv_to-be-replaced" principal_name = "webAppPrincipalName" admin_principal_name = "adminAppPrincipalName" +user = "managedIdentityName" def get_secrets_from_kv(kv_name, secret_name): credential = DefaultAzureCredential() @@ -57,7 +58,6 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): postgres_details = json.loads(get_secrets_from_kv(key_vault_name, "AZURE-POSTGRESQL-INFO")) host = postgres_details.get("host", "") -user = "wpvykucviclze-managed-identity" dbname = postgres_details.get("dbname", "") password = postgres_details.get("password", "") diff --git a/scripts/run_create_table_script.sh b/scripts/run_create_table_script.sh index eeef0934e..31bc4cf61 100644 --- a/scripts/run_create_table_script.sh +++ b/scripts/run_create_table_script.sh @@ -10,6 +10,7 @@ resourceGroup="$3" serverName="$4" webAppPrincipalName="$5" adminAppPrincipalName="$6" +managedIdentityName="$7" echo "Script Started" @@ -31,6 +32,7 @@ echo "Download completed" sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "create_postgres_tables.py" sed -i "s/webAppPrincipalName/${webAppPrincipalName}/g" "create_postgres_tables.py" sed -i "s/adminAppPrincipalName/${adminAppPrincipalName}/g" "create_postgres_tables.py" +sed -i "s/managedIdentityName/${managedIdentityName}/g" "create_postgres_tables.py" pip install -r requirements.txt From c9fe9ef60deca21649488f4b9efe39ab67b6e0b8 Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Fri, 29 Nov 2024 16:20:06 +0530 Subject: [PATCH 12/13] Updated code for testing --- scripts/data_scripts/create_postgres_tables.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/data_scripts/create_postgres_tables.py b/scripts/data_scripts/create_postgres_tables.py index 247cb30ca..fa346e202 100644 --- a/scripts/data_scripts/create_postgres_tables.py +++ b/scripts/data_scripts/create_postgres_tables.py @@ -57,8 +57,8 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): ) postgres_details = json.loads(get_secrets_from_kv(key_vault_name, "AZURE-POSTGRESQL-INFO")) -host = postgres_details.get("host", "") -dbname = postgres_details.get("dbname", "") +host = postgres_details.get("POSTGRESQL_HOST", "") +dbname = postgres_details.get("POSTGRESQL_DATABASE", "") password = postgres_details.get("password", "") # Acquire the access token @@ -73,8 +73,9 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): cursor = conn.cursor() grant_permissions(cursor, dbname, "public", principal_name) -grant_permissions(cursor, dbname, "public", admin_principal_name) conn.commit() +# grant_permissions(cursor, dbname, "public", admin_principal_name) +# conn.commit() # Drop and recreate the conversations table cursor.execute("DROP TABLE IF EXISTS conversations") From 9b0ae47720523c14e9a9fa450e335c52d435a8c1 Mon Sep 17 00:00:00 2001 From: Prajwal D C Date: Fri, 29 Nov 2024 16:27:50 +0530 Subject: [PATCH 13/13] Test script --- scripts/data_scripts/create_postgres_tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/data_scripts/create_postgres_tables.py b/scripts/data_scripts/create_postgres_tables.py index fa346e202..8ae2f7c6e 100644 --- a/scripts/data_scripts/create_postgres_tables.py +++ b/scripts/data_scripts/create_postgres_tables.py @@ -57,8 +57,8 @@ def grant_permissions(cursor, dbname, schema_name, principal_name): ) postgres_details = json.loads(get_secrets_from_kv(key_vault_name, "AZURE-POSTGRESQL-INFO")) -host = postgres_details.get("POSTGRESQL_HOST", "") -dbname = postgres_details.get("POSTGRESQL_DATABASE", "") +host = postgres_details.get("host", "") +dbname = postgres_details.get("database", "") password = postgres_details.get("password", "") # Acquire the access token