diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index fded4435..00000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# Lines starting with '#' are comments. -# Each line is a file pattern followed by one or more owners. - -# These owners will be the default owners for everything in the repo. -* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft @dongbumlee \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 882ebd79..00000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: bug -assignees: '' - ---- - -# Describe the bug -A clear and concise description of what the bug is. - -# Expected behavior -A clear and concise description of what you expected to happen. - -# How does this bug make you feel? -_Share a gif from [giphy](https://giphy.com/) to tells us how you'd feel_ - ---- - -# Debugging information - -## Steps to reproduce -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -## Screenshots -If applicable, add screenshots to help explain your problem. - -## Logs - -If applicable, add logs to help the engineer debug the problem. - ---- - -# Tasks - -_To be filled in by the engineer picking up the issue_ - -- [ ] Task 1 -- [ ] Task 2 -- [ ] ... diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 3496fc82..00000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: enhancement -assignees: '' - ---- - -# Motivation - -A clear and concise description of why this feature would be useful and the value it would bring. -Explain any alternatives considered and why they are not sufficient. - -# How would you feel if this feature request was implemented? - -_Share a gif from [giphy](https://giphy.com/) to tells us how you'd feel. Format: ![alt_text](https://media.giphy.com/media/xxx/giphy.gif)_ - -# Requirements - -A list of requirements to consider this feature delivered -- Requirement 1 -- Requirement 2 -- ... - -# Tasks - -_To be filled in by the engineer picking up the issue_ - -- [ ] Task 1 -- [ ] Task 2 -- [ ] ... diff --git a/.github/ISSUE_TEMPLATE/subtask.md b/.github/ISSUE_TEMPLATE/subtask.md deleted file mode 100644 index 9f86c843..00000000 --- a/.github/ISSUE_TEMPLATE/subtask.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -name: Sub task -about: A sub task -title: '' -labels: subtask -assignees: '' - ---- - -Required by - -# Description - -A clear and concise description of what this subtask is. - -# Tasks - -_To be filled in by the engineer picking up the subtask - -- [ ] Task 1 -- [ ] Task 2 -- [ ] ... diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 22b04b72..00000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,37 +0,0 @@ -## Purpose - -* ... - -## Does this introduce a breaking change? - - -- [ ] Yes -- [ ] No - - - -## Golden Path Validation -- [ ] I have tested the primary workflows (the "golden path") to ensure they function correctly without errors. - -## Deployment Validation -- [ ] I have validated the deployment process successfully and all services are running as expected with this change. - -## What to Check -Verify that the following are valid -* ... - -## Other Information - \ No newline at end of file diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 00000000..a28b9b6e --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,181 @@ +name: Deploy Resources + +on: + push: + branches: + - main # Adjust this to the branch you want to trigger the deployment on + schedule: + - cron: '0 6,18 * * *' # Runs at 6:00 AM and 6:00 PM GMT + + +jobs: + deploy: + runs-on: windows-latest # Use a Windows runner for PowerShell scripts + + steps: + - name: Checkout Code + uses: actions/checkout@v3 # Checks out your repository + # Install Azure CLI + - name: Install Azure CLI + shell: pwsh + run: | + Invoke-WebRequest -Uri https://aka.ms/installazurecliwindows -OutFile AzureCLI.msi + Start-Process msiexec.exe -ArgumentList '/I AzureCLI.msi /quiet' -Wait + + # Install kubectl (Windows method) + - name: Install kubectl + shell: pwsh + run: | + Invoke-WebRequest -Uri https://dl.k8s.io/release/v1.28.0/bin/windows/amd64/kubectl.exe -OutFile kubectl.exe + Move-Item -Path ./kubectl.exe -Destination "C:\kubectl.exe" + [Environment]::SetEnvironmentVariable('PATH', $env:PATH + ';C:\', [System.EnvironmentVariableTarget]::Machine) + + + # Install Helm (Windows method) + - name: Install Helm + shell: pwsh + run: | + Invoke-WebRequest -Uri https://get.helm.sh/helm-v3.13.0-windows-amd64.zip -OutFile helm.zip + Expand-Archive helm.zip -DestinationPath helm + Move-Item -Path ./helm/windows-amd64/helm.exe -Destination "C:\helm.exe" + [Environment]::SetEnvironmentVariable('PATH', $env:PATH + ';C:\', [System.EnvironmentVariableTarget]::Machine) + + + - name: Set Docker environment variables + run: echo "DOCKER_BUILDKIT=0" >> $GITHUB_ENV + + # Set up Docker + - name: Set up Docker + uses: docker/setup-buildx-action@v2 + with: + driver: docker + + - name: Setup PowerShell + shell: pwsh + run: | + $PSVersionTable.PSVersion + + - name: Run Deployment Script with Input + shell: pwsh + run: | + cd Deployment + $input = @" + ${{ secrets.AZURE_SUBSCRIPTION_ID }} + CanadaCentral + WestUS3 + ${{ secrets.EMAIL }} + yes + "@ + $input | pwsh ./resourcedeployment.ps1 + echo "Resource Group Name is ${{ env.rg_name }}" + echo "Kubernetes resource group are ${{ env.krg_name }}" + env: + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + - name: Cleanup Resource Group + if: always() # Ensures this step runs even if the deployment fails + shell: pwsh + run: | + az login --service-principal --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} + az group delete --name ${{ env.rg_name }} --yes --no-wait + az group delete --name ${{ env.krg_name }} --yes --no-wait + env: + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + + - name: Wait for Resource Deletion to Complete + shell: pwsh + run: | + $retries = 0 + $maxRetries = 3 + $sleepIntervals = @(700, 200, 200) + + while ($retries -lt $maxRetries) { + $rgStatus = az group exists --name ${{ env.rg_name }} + $krgStatus = az group exists --name ${{ env.krg_name }} + + + # if (-not $rgStatus -and -not $krgStatus) { + # Write-Host "Both resource groups deleted successfully." + # break + # } + if ($rgStatus -eq "false" -and $krgStatus -eq "false") { + Write-Host "Both resource groups deleted successfully." + break + } + + $retries++ + if ($retries -eq $maxRetries) { + Write-Host "Resource groups deletion not confirmed after $maxRetries attempts. Exiting." + exit 1 + } + + Write-Host "Resource groups still exist. Retrying in $($sleepIntervals[$retries - 1]) seconds..." + Start-Sleep -Seconds $sleepIntervals[$retries - 1] + } + + - name: Purging the Resources + if: success() + shell: pwsh + run: | + # Set variables using GitHub Actions environment values + $solutionPrefix = "${{ env.SOLUTION_PREFIX }}" + $subscriptionId = "${{ secrets.AZURE_SUBSCRIPTION_ID }}" + $resourceGroupName = "${{ env.rg_name }}" + + $openai_name = "openaiservice-$solutionPrefix" + $cognitiveservice_name = "cognitiveservice-$solutionPrefix" + + # Debug: Print resource names + Write-Host "Purging OpenAI resource: $openai_name" + Write-Host "Purging CognitiveService Account: $cognitiveservice_name" + + # Construct resource IDs + $openaiResourceId = "/subscriptions/$subscriptionId/providers/Microsoft.CognitiveServices/locations/westus3/resourceGroups/$resourceGroupName/deletedAccounts/$openai_name" + $cognitiveResourceId = "/subscriptions/$subscriptionId/providers/Microsoft.CognitiveServices/locations/eastus/resourceGroups/$resourceGroupName/deletedAccounts/$cognitiveservice_name" + + # Debug: Print constructed resource IDs + Write-Host "Command to purge OpenAI resource: az resource delete --ids `"$openaiResourceId`" --verbose" + Write-Host "Command to purge CognitiveService Account: az resource delete --ids `"$cognitiveResourceId`" --verbose" + # Purge OpenAI Resource + az resource delete --ids $openaiResourceId --verbose + if (-not $?) { + Write-Host "Failed to purge OpenAI resource: $openaiResourceId" + } + + # Purge CognitiveService Account + + + az resource delete --ids $cognitiveResourceId --verbose + if (-not $?) { + Write-Host "Failed to purge CognitiveService Account." + } + + + - name: Send Notification on Failure + if: failure() + shell: pwsh + run: | + # Define the RUN_URL variable + $RUN_URL = "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + # Construct the email body using a Here-String + $EMAIL_BODY = @" + { + "body": "

Dear Team,

The Document Knowledge Mining Automation process encountered an issue.

Build URL: $RUN_URL

Please investigate promptly.

Best regards,
Your Automation Team

" + } + "@ + + # Send the notification with error handling + try { + curl -X POST "${{ secrets.LOGIC_APP_URL }}" ` + -H "Content-Type: application/json" ` + -d "$EMAIL_BODY" + } catch { + Write-Output "Failed to send notification." + } + \ No newline at end of file diff --git a/.github/workflows/pr-title-checker.yml b/.github/workflows/pr-title-checker.yml deleted file mode 100644 index 5cbbae1a..00000000 --- a/.github/workflows/pr-title-checker.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "pr-title-checker" - -on: - pull_request_target: - types: - - opened - - edited - - synchronize - merge_group: - -permissions: - pull-requests: read - -jobs: - main: - name: Validate PR title - runs-on: ubuntu-latest - if: ${{ github.event_name != 'merge_group' }} - steps: - - uses: amannn/action-semantic-pull-request@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/stale-bot.yml b/.github/workflows/stale-bot.yml deleted file mode 100644 index e31059ab..00000000 --- a/.github/workflows/stale-bot.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: 'Close stale issues and PRs' -on: - schedule: - - cron: '30 1 * * *' - -permissions: - contents: write - issues: write - pull-requests: write - -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v9 - with: - stale-issue-message: 'This issue is stale because it has been open 180 days with no activity. Remove stale label or comment or this will be closed in 30 days.' - days-before-stale: 180 - days-before-close: 30 diff --git a/App/kernel-memory/service/Abstractions/Constants.cs b/App/kernel-memory/service/Abstractions/Constants.cs index d2e516f7..53d5535e 100644 --- a/App/kernel-memory/service/Abstractions/Constants.cs +++ b/App/kernel-memory/service/Abstractions/Constants.cs @@ -163,5 +163,4 @@ public static class Summary // Standard prompt names public const string PromptNamesSummarize = "summarize"; public const string PromptNamesAnswerWithFacts = "answer-with-facts"; - public const string PromptNamesExtractKeywords = "extract-keywords"; } diff --git a/App/kernel-memory/service/Core/Core.csproj b/App/kernel-memory/service/Core/Core.csproj index c3ab4dd3..71cec67b 100644 --- a/App/kernel-memory/service/Core/Core.csproj +++ b/App/kernel-memory/service/Core/Core.csproj @@ -42,7 +42,6 @@ - diff --git a/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs b/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs index df66dedf..b5f95c9b 100644 --- a/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs @@ -11,7 +11,6 @@ using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; using static Microsoft.KernelMemory.Pipeline.DataPipeline; -using Microsoft.KernelMemory.Prompts; namespace Microsoft.KernelMemory.Handlers; @@ -22,13 +21,11 @@ public sealed class KeywordExtractingHandler : IPipelineStepHandler private readonly IPipelineOrchestrator _orchestrator; private readonly Kernel _kernel; private readonly KernelMemoryConfig? _config = null; - private readonly string _extractKeywordPrompt; public KeywordExtractingHandler( string stepName, IPipelineOrchestrator orchestrator, KernelMemoryConfig config = null, - IPromptProvider? promptProvider = null, ILoggerFactory? loggerFactory = null ) { @@ -37,10 +34,6 @@ public KeywordExtractingHandler( this._orchestrator = orchestrator; this._config = config; - promptProvider ??= new EmbeddedPromptProvider(); - - this._extractKeywordPrompt = promptProvider.ReadPrompt(Constants.PromptNamesExtractKeywords); - //init Semantic Kernel this._kernel = Kernel.CreateBuilder() .AddAzureOpenAIChatCompletion(deploymentName: (string)this._config.Services["AzureOpenAIText"]["Deployment"], @@ -86,7 +79,30 @@ public KeywordExtractingHandler( var chat = this._kernel.GetRequiredService(); var chatHistory = new ChatHistory(); - chatHistory.AddSystemMessage(this._extractKeywordPrompt); + var systemMessage = """ + You are an assistant to analyze Content and Extract Tags by Content. + [EXTRACT TAGS RULES] + IT SHOULD BE A LIST OF DICTIONARIES WITH CATEGORY AND TAGS + TAGS SHOULD BE CATEGORY SPECIFIC + TAGS SHOULD BE A LIST OF STRINGS + TAGS COUNT CAN BE UP TO 10 UNDER A CATEGORY + CATEGORY COUNT CAN BE UP TO 10 + DON'T ADD ANY MARKDOWN EXPRESSION IN YOUR RESPONSE + [END RULES] + + [EXAMPLE] + [ + { + [category1": ["tag1", "tag2", "tag3"] + }, + { + "category2": ["tag1", "tag2", "tag3"] + } + ] + [END EXAMPLE] + """; + + chatHistory.AddSystemMessage(systemMessage); chatHistory.AddUserMessage($"Extract tags from this content : {extactedFileContent} \n The format should be Json but Markdown expression."); var executionParam = new PromptExecutionSettings() diff --git a/App/kernel-memory/service/Core/Prompts/extract-keywords.txt b/App/kernel-memory/service/Core/Prompts/extract-keywords.txt index 9dd6485c..72a3e9b8 100644 --- a/App/kernel-memory/service/Core/Prompts/extract-keywords.txt +++ b/App/kernel-memory/service/Core/Prompts/extract-keywords.txt @@ -1,25 +1,21 @@ -You are an assistant to analyze Content and Extract Tags by Content. -[EXTRACT TAGS RULES] -IT SHOULD BE A LIST OF DICTIONARIES WITH CATEGORY AND TAGS -TAGS SHOULD BE CATEGORY SPECIFIC -TAGS SHOULD BE A LIST OF STRINGS -TAGS COUNT CAN BE UP TO 10 UNDER A CATEGORY +[EXTRACT KEYWORDS RULES] +IT SHOULD BE A LIST OF DICTIONARIES WITH CATEGORY AND KEYWORDS +KEYWORDS SHOULD BE CATEGORY SPECIFIC +KEYWORDS SHOULD BE A LIST OF STRINGS +KEYWORDS COUNT CAN BE UP TO 50 CATEGORY COUNT CAN BE UP TO 10 -DON'T ADD ANY MARKDOWN EXPRESSION IN YOUR RESPONSE [END RULES] + [EXAMPLE] [ { - "category1": ["tag1", "tag2", "tag3"] + "category1": ["keyword1", "keyword2", "keyword3"] }, { - "category2": ["tag1", "tag2", "tag3"] + "category2": ["keyword1", "keyword2", "keyword3"] } ] [END EXAMPLE] -Extract Tags from this : +Extract Keywords from this : {{$input}} - - - diff --git a/Deployment/main.bicep b/Deployment/main.bicep index afbe4912..29fa5f4a 100644 --- a/Deployment/main.bicep +++ b/Deployment/main.bicep @@ -158,6 +158,7 @@ module gs_appconfig 'bicep/azureappconfigservice.bicep' = { // return all resource names as a output output gs_resourcegroup_name string = 'rg-${resourceprefix_name}${resourceprefix}' +output gs_solution_prefix string = '${resourceprefix_name}${resourceprefix}' output gs_storageaccount_name string = gs_storageaccount.outputs.storageAccountName output gs_azsearch_name string = gs_azsearch.outputs.searchServiceName diff --git a/Deployment/resourcedeployment.ps1 b/Deployment/resourcedeployment.ps1 index 35a5342a..a56a061f 100644 --- a/Deployment/resourcedeployment.ps1 +++ b/Deployment/resourcedeployment.ps1 @@ -141,8 +141,19 @@ $modelLocation = $params.modelLocation $email = $params.email function LoginAzure([string]$subscriptionID) { - Write-Host "Log in to Azure.....`r`n" -ForegroundColor Yellow - az login + Write-Host "Log in to Azure.....`r`n" -ForegroundColor Yellow + if ($env:CI -eq "true"){ + + az login --service-principal ` + --username $env:AZURE_CLIENT_ID ` + --password $env:AZURE_CLIENT_SECRET ` + --tenant $env:AZURE_TENANT_ID + write-host "CI deployment mode" + } + else{ + az login + write-host "manual deployment mode" + } az account set --subscription $subscriptionID Write-Host "Switched subscription to '$subscriptionID' `r`n" -ForegroundColor Yellow } @@ -192,9 +203,12 @@ function DeployAzureResources([string]$location, [string]$modelLocation) { function DisplayResult([pscustomobject]$jsonString) { $resourcegroupName = $jsonString.properties.outputs.gs_resourcegroup_name.value + $solutionPrefix = $jsonString.properties.outputs.gs_solution_prefix.value + $storageAccountName = $jsonString.properties.outputs.gs_storageaccount_name.value $azsearchServiceName = $jsonString.properties.outputs.gs_azsearch_name.value $aksName = $jsonString.properties.outputs.gs_aks_name.value + $containerRegistryName = $jsonString.properties.outputs.gs_containerregistry_name.value $azcognitiveserviceName = $jsonString.properties.outputs.gs_azcognitiveservice_name.value $azopenaiServiceName = $jsonString.properties.outputs.gs_openaiservice_name.value @@ -215,6 +229,9 @@ function DisplayResult([pscustomobject]$jsonString) { Write-Host "* Azure Storage Account " -ForegroundColor Yellow -NoNewline; Write-Host "$storageAccountName" -ForegroundColor Green Write-Host "* Azure Cosmos DB " -ForegroundColor Yellow -NoNewline; Write-Host "$azcosmosDBName" -ForegroundColor Green Write-Host "* Azure App Configuration Endpoint " -ForegroundColor Yellow -NoNewline; Write-Host "$azappConfigEndpoint" -ForegroundColor Green + Write-Output "rg_name=$resourcegroupName" >> $Env:GITHUB_ENV + + Write-Output "SOLUTION_PREFIX=$solutionPrefix" >> $Env:GITHUB_ENV } # Function to replace placeholders in a template with actual values @@ -590,6 +607,8 @@ try { Write-Host "Getting the Kubernetes resource group..." -ForegroundColor Cyan $aksResourceGroupName = $(az aks show --resource-group $deploymentResult.ResourceGroupName --name $deploymentResult.AksName --query nodeResourceGroup --output tsv) Write-Host "Kubernetes resource group: $aksResourceGroupName" -ForegroundColor Green + Write-Output "krg_name=$aksResourceGroupName" >> $Env:GITHUB_ENV + } catch { Write-Host "Failed to get the Kubernetes resource group." -ForegroundColor Red @@ -716,6 +735,7 @@ try { Write-Host "Upgrading node pool: $nodePool" -ForegroundColor Cyan Write-Host "Node pool $nodePool upgrade initiated." -ForegroundColor Green az aks nodepool upgrade --resource-group $deploymentResult.ResourceGroupName --cluster-name $deploymentResult.AksName --name $nodePool + } } catch {