Merge pull request #41 from Fr4nc3/bicepdefaults

fix: Create Pgsql Database and run the scripts to create the tables and add extensions

Author: Fr4nc3

code/tests/functional/app_config.py

@@ -7,6 +7,7 @@
 logger = logging.getLogger(__name__)
 encoded_account_key = str(base64.b64encode(b"some-blob-account-key"), "utf-8")
 
+
 class AppConfig:
     before_config: dict[str, str] = {}
     config: dict[str, str | None] = {

infra/app/storekeys.bicep

@@ -110,7 +110,7 @@ resource postgresInfoSecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = if
       ? string({
           user: postgresDatabaseAdminUserName
           dbname: postgresDatabaseName
-          host: '${postgresServerName}.postgres.database.azure.com'
+          host: postgresServerName
           password: postgresDatabaseAdminPassword
         })
       : ''

infra/core/database/deploy_create_table_script.bicep

@@ -0,0 +1,27 @@
+@description('Specifies the location for resources.')
+param solutionLocation string
+
+param baseUrl string
+param keyVaultName string
+param identity string
+param postgresSqlServerName string
+
+resource create_index 'Microsoft.Resources/deploymentScripts@2020-10-01' = {
+  kind:'AzureCLI'
+  name: 'create_postgres_table'
+  location: solutionLocation // Replace with your desired location
+  identity: {
+    type: 'UserAssigned'
+    userAssignedIdentities: {
+      '${identity}' : {}
+    }
+  }
+  properties: {
+    azCliVersion: '2.52.0'
+    primaryScriptUri: '${baseUrl}scripts/run_create_table_script.sh'
+    arguments: '${baseUrl} ${keyVaultName} ${resourceGroup().name} ${postgresSqlServerName}' // Specify any arguments for the script
+    timeout: 'PT1H' // Specify the desired timeout duration
+    retentionInterval: 'PT1H' // Specify the desired retention interval
+    cleanupPreference:'OnSuccess'
+  }
+}

infra/core/database/postgresdb.bicep

@@ -1,5 +1,7 @@
 param solutionName string
 param solutionLocation string
+param managedIdentityObjectId string
+param managedIdentityObjectName string
 @description('The name of the SQL logical server.')
 param serverName string = '${solutionName}-postgres'
 
@@ -35,7 +37,11 @@ resource serverName_resource 'Microsoft.DBforPostgreSQL/flexibleServers@2023-12-
     version: version
     administratorLogin: administratorLogin
     administratorLoginPassword: administratorLoginPassword
-
+    authConfig: {
+      tenantId: subscription().tenantId
+      activeDirectoryAuth: 'Enabled'
+      passwordAuth: 'Enabled'
+    }
     highAvailability: {
       mode: 'Disabled'
     }
@@ -53,6 +59,34 @@ resource serverName_resource 'Microsoft.DBforPostgreSQL/flexibleServers@2023-12-
   }
 }
 
+resource delayScript 'Microsoft.Resources/deploymentScripts@2020-10-01' = {
+  name: 'waitForServerReady'
+  location: resourceGroup().location
+  kind: 'AzurePowerShell'
+  properties: {
+    azPowerShellVersion: '3.0'
+    scriptContent: 'start-sleep -Seconds 180'
+    cleanupPreference: 'Always'
+    retentionInterval: 'PT1H'
+  }
+  dependsOn: [
+    serverName_resource
+  ]
+}
+
+resource azureADAdministrator 'Microsoft.DBforPostgreSQL/flexibleServers/administrators@2022-12-01' = {
+  parent: serverName_resource
+  name: managedIdentityObjectId
+  properties: {
+    principalType: 'SERVICEPRINCIPAL'
+    principalName: managedIdentityObjectName
+    tenantId: subscription().tenantId
+  }
+  dependsOn: [
+    delayScript
+  ]
+}
+
 // resource serverName_firewallrules 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2021-06-01' = [for rule in firewallrules: {
 //   parent: serverName_resource
 //   name: rule.Name
@@ -71,7 +105,7 @@ resource firewall_all 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules@2
     endIpAddress: '255.255.255.255'
   }
   dependsOn: [
-    serverName_resource
+    delayScript
   ]
 }
 
@@ -83,15 +117,15 @@ resource firewall_azure 'Microsoft.DBforPostgreSQL/flexibleServers/firewallRules
     endIpAddress: '0.0.0.0'
   }
   dependsOn: [
-    firewall_all
+    delayScript
   ]
 }
 
 resource configurations 'Microsoft.DBforPostgreSQL/flexibleServers/configurations@2023-12-01-preview' = {
   name: 'azure.extensions'
   parent: serverName_resource
   properties: {
-    value: 'vector'
+    value: 'pg_diskann'
     source: 'user-override'
   }
   dependsOn: [
@@ -102,7 +136,7 @@ resource configurations 'Microsoft.DBforPostgreSQL/flexibleServers/configuration
 
 output postgresDbOutput object = {
   postgresSQLName: serverName_resource.name
-  postgreSQLServerName: serverName_resource.name
+  postgreSQLServerName: '${serverName_resource.name}.postgres.database.azure.com'
   postgreSQLDatabaseName: 'postgres'
   postgreSQLDbUser: administratorLogin
   postgreSQLDbPwd: administratorLoginPassword

infra/core/security/keyvault.bicep

@@ -2,6 +2,7 @@ metadata description = 'Creates an Azure Key Vault.'
 param name string
 param location string = resourceGroup().location
 param tags object = {}
+param managedIdentityObjectId string
 
 param principalId string = ''
 
@@ -12,18 +13,43 @@ resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = {
   properties: {
     tenantId: subscription().tenantId
     sku: { family: 'A', name: 'standard' }
-    accessPolicies: !empty(principalId) 
-     ? [
-      {
-        objectId: principalId
-        permissions: { secrets: [ 'get', 'list' ] }
-        tenantId: subscription().tenantId
-      }
-    ] 
-     : []
+    accessPolicies: !empty(principalId)
+      ? [
+        {
+          objectId: principalId
+          permissions: { secrets: [ 'get', 'list' ] }
+          tenantId: subscription().tenantId
+        }, {
+          objectId: managedIdentityObjectId
+          permissions: { secrets: [ 'get', 'list' ] }
+          tenantId: subscription().tenantId
+        }
+      ]
+      : [
+        {
+          objectId: managedIdentityObjectId
+          permissions: { secrets: [ 'get', 'list' ] }
+          tenantId: subscription().tenantId
+        }
+      ]
   }
 }
 
+// @description('This is the built-in Key Vault Administrator role.')
+// resource kvAdminRole 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = {
+//   scope: resourceGroup()
+//   name: '00482a5a-887f-4fb3-b363-3b7fe8e74483'
+// }
+
+// resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+//   name: guid(resourceGroup().id, managedIdentityObjectId, kvAdminRole.id)
+//   properties: {
+//     principalId: managedIdentityObjectId
+//     roleDefinitionId:kvAdminRole.id
+//     principalType: 'ServicePrincipal'
+//   }
+// }
+
 output endpoint string = keyVault.properties.vaultUri
 output name string = keyVault.name
-output id string = keyVault.id
+output id string = keyVault.id

infra/core/security/managed-identity.bicep

@@ -0,0 +1,43 @@
+// ========== Managed Identity ========== //
+targetScope = 'resourceGroup'
+
+@minLength(3)
+@maxLength(15)
+@description('Solution Name')
+param solutionName string
+
+@description('Solution Location')
+param solutionLocation string
+
+@description('Name')
+param miName string = '${ solutionName }-managed-identity'
+
+resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
+  name: miName
+  location: solutionLocation
+  tags: {
+    app: solutionName
+    location: solutionLocation
+  }
+}
+
+@description('This is the built-in owner role. See https://docs.microsoft.com/azure/role-based-access-control/built-in-roles#owner')
+resource ownerRoleDefinition 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = {
+  scope: resourceGroup()
+  name: '8e3af657-a8ff-443c-a75c-2fe8c4bcb635'
+}
+
+resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  name: guid(resourceGroup().id, managedIdentity.id, ownerRoleDefinition.id)
+  properties: {
+    principalId: managedIdentity.properties.principalId
+    roleDefinitionId:  ownerRoleDefinition.id
+    principalType: 'ServicePrincipal'
+  }
+}
+
+output managedIdentityOutput object = {
+  id: managedIdentity.id
+  objectId: managedIdentity.properties.principalId
+  name: miName
+}

infra/main.bicep

@@ -148,7 +148,7 @@ param azureOpenAIVisionModelCapacity int = 10
   'langchain'
   'prompt_flow'
 ])
-param orchestrationStrategy string = 'openai_function'
+param orchestrationStrategy string = 'semantic_kernel'
 
 @description('Chat conversation type: custom or byod.')
 @allowed([
@@ -315,20 +315,14 @@ param azureCosmosDBAccountName string = 'cosmos-${resourceToken}'
 @description('Azure Postgres DB Account Name')
 param azurePostgresDBAccountName string = 'postgres-${resourceToken}'
 
-@description('Whether or not to enable chat history')
-@allowed([
-  'true'
-  'false'
-])
-param chatHistoryEnabled string = 'true'
-
 var blobContainerName = 'documents'
 var queueName = 'doc-processing'
 var clientKey = '${uniqueString(guid(subscription().id, deployment().name))}${newGuidString}'
 var eventGridSystemTopicName = 'doc-processing'
 var tags = { 'azd-env-name': environmentName }
 var rgName = 'rg-${environmentName}'
 var keyVaultName = 'kv-${resourceToken}'
+var baseUrl = 'https://raw.githubusercontent.com/Fr4nc3/chat-with-your-data-solution-accelerator/bicepdefaults/'
 var azureOpenAIModelInfo = string({
   model: azureOpenAIModel
   modelName: azureOpenAIModelName
@@ -350,6 +344,16 @@ resource rg 'Microsoft.Resources/resourceGroups@2021-04-01' = {
   tags: tags
 }
 
+// ========== Managed Identity ========== //
+module managedIdentityModule './core/security/managed-identity.bicep' = if (databaseType == 'postgres') {
+  name: 'deploy_managed_identity'
+  params: {
+    solutionName: resourceToken
+    solutionLocation: location
+  }
+  scope: rg
+}
+
 module cosmosDBModule './core/database/cosmosdb.bicep' = if (databaseType == 'cosmos') {
   name: 'deploy_cosmos_db'
   params: {
@@ -364,8 +368,10 @@ module postgresDBModule './core/database/postgresdb.bicep' = if (databaseType ==
   params: {
     solutionName: azurePostgresDBAccountName
     solutionLocation: 'eastus2'
+    managedIdentityObjectId: managedIdentityModule.outputs.managedIdentityOutput.objectId
+    managedIdentityObjectName: managedIdentityModule.outputs.managedIdentityOutput.name
   }
-  scope: resourceGroup(resourceGroup().name)
+  scope: rg
 }
 
 // Store secrets in a keyvault
@@ -377,6 +383,7 @@ module keyvault './core/security/keyvault.bicep' = if (useKeyVault || authType =
     location: location
     tags: tags
     principalId: principalId
+    managedIdentityObjectId: managedIdentityModule.outputs.managedIdentityOutput.objectId
   }
 }
 
@@ -764,7 +771,6 @@ module web_docker './app/web.bicep' = if (hostingModel == 'container') {
       ORCHESTRATION_STRATEGY: orchestrationStrategy
       CONVERSATION_FLOW: conversationFlow
       LOGLEVEL: logLevel
-      CHAT_HISTORY_ENABLED: chatHistoryEnabled
 
       // Add database type to settings
       AZURE_DATABASE_TYPE: databaseType
@@ -849,7 +855,6 @@ module adminweb './app/adminweb.bicep' = if (hostingModel == 'code') {
       FUNCTION_KEY: clientKey
       ORCHESTRATION_STRATEGY: orchestrationStrategy
       LOGLEVEL: logLevel
-      CHAT_HISTORY_ENABLED: chatHistoryEnabled
     }
   }
 }
@@ -923,7 +928,6 @@ module adminweb_docker './app/adminweb.bicep' = if (hostingModel == 'container')
       FUNCTION_KEY: clientKey
       ORCHESTRATION_STRATEGY: orchestrationStrategy
       LOGLEVEL: logLevel
-      CHAT_HISTORY_ENABLED: chatHistoryEnabled
     }
   }
 }
@@ -1226,6 +1230,19 @@ module machineLearning 'app/machinelearning.bicep' = if (orchestrationStrategy =
   }
 }
 
+module createIndex './core/database/deploy_create_table_script.bicep' =  if (databaseType == 'postgres') {
+  name : 'deploy_create_table_script'
+  params:{
+    solutionLocation: location
+    identity:managedIdentityModule.outputs.managedIdentityOutput.id
+    baseUrl:baseUrl
+    keyVaultName:keyvault.outputs.name
+    postgresSqlServerName: postgresDBModule.outputs.postgresDbOutput.postgresSQLName
+  }
+  scope: rg
+  dependsOn:[keyvault, postgresDBModule, storekeys]
+}
+
 output APPLICATIONINSIGHTS_CONNECTION_STRING string = monitoring.outputs.applicationInsightsConnectionString
 output AZURE_APP_SERVICE_HOSTING_MODEL string = hostingModel
 output AZURE_BLOB_STORAGE_INFO string = replace(azureBlobStorageInfo, '$STORAGE_ACCOUNT_KEY','')

infra/main.bicepparam

@@ -19,7 +19,6 @@ param orchestrationStrategy = readEnvironmentVariable('ORCHESTRATION_STRATEGY',
 param logLevel = readEnvironmentVariable('LOGLEVEL', 'INFO')
 param recognizedLanguages = readEnvironmentVariable('AZURE_SPEECH_RECOGNIZER_LANGUAGES', 'en-US,fr-FR,de-DE,it-IT')
 param conversationFlow = readEnvironmentVariable('CONVERSATION_FLOW', 'custom')
-param chatHistoryEnabled = readEnvironmentVariable('CHAT_HISTORY_ENABLED', 'true')
 
 //Azure Search
 param azureSearchFieldId = readEnvironmentVariable('AZURE_SEARCH_FIELDS_ID', 'id')