feat: Added EXP Changes, Model updates & End to End tests

.env.sample

@@ -64,6 +64,7 @@ AZURE_KEY_VAULT_ENDPOINT=
 CONVERSATION_FLOW=
 # Chat History CosmosDB Integration Settings
 AZURE_COSMOSDB_ACCOUNT_NAME=
+AZURE_COSMOSDB_ACCOUNT_KEY=
 AZURE_COSMOSDB_DATABASE_NAME=
 AZURE_COSMOSDB_CONVERSATIONS_CONTAINER_NAME=
 AZURE_COSMOSDB_ENABLE_FEEDBACK=

.github/workflows/ci.yml

@@ -103,7 +103,7 @@ jobs:
           imageName: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator
           cacheFrom: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator
           imageTag: ${{ env.imageTag }}
-          runCmd: make ci && make deploy
+          runCmd: export optional_args="./code/tests" && make ci && make deploy
           refFilterForPush: refs/heads/${{ github.event_name == 'schedule' && 'main' || github.ref_name }}
           env: |
             AZURE_CLIENT_ID

.github/workflows/test-automation.yml

@@ -0,0 +1,106 @@
+name: Test Automation CWYD
+on:
+  push:
+    branches:
+      - main
+      - dev
+    paths:
+      - 'tests/e2e-test/**'
+  schedule:
+    - cron: '0 13 * * 3'  # Runs at 1 PM UTC once a week on Wednesday
+  workflow_dispatch:
+
+env:
+    web_url: ${{ vars.CWYD_WEB_URL }}
+    admin_url: ${{ vars.CWYD_ADMIN_URL }}
+    accelerator_name: "Chat with your Data"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.13'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tests/e2e-test/requirements.txt
+
+      - name: Ensure browsers are installed
+        run: python -m playwright install --with-deps chromium
+
+      - name: Run tests(1)
+        id: test1
+        run: |
+          xvfb-run pytest --headed --html=report/report.html --self-contained-html
+        working-directory: tests/e2e-test
+        continue-on-error: true
+
+      - name: Sleep for 30 seconds
+        if: ${{ steps.test1.outcome == 'failure' }}
+        run: sleep 30s
+        shell: bash
+
+      - name: Run tests(2)
+        id: test2
+        if: ${{ steps.test1.outcome == 'failure' }}
+        run: |
+          xvfb-run pytest --headed --html=report/report.html --self-contained-html
+        working-directory: tests/e2e-test
+        continue-on-error: true
+
+      - name: Sleep for 60 seconds
+        if: ${{ steps.test2.outcome == 'failure' }}
+        run: sleep 60s
+        shell: bash
+
+      - name: Run tests(3)
+        id: test3
+        if: ${{ steps.test2.outcome == 'failure' }}
+        run: |
+          xvfb-run pytest --headed --html=report/report.html --self-contained-html
+        working-directory: tests/e2e-test
+
+      - name: Upload test report
+        id: upload_report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: cwyd-test-report
+          path: tests/e2e-test/report/*
+
+      - name: Send Notification
+        if: always()
+        run: |
+          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          REPORT_URL=${{ steps.upload_report.outputs.artifact-url }}
+          IS_SUCCESS=${{ steps.test1.outcome == 'success' || steps.test2.outcome == 'success' || steps.test3.outcome == 'success' }}
+          # Construct the email body
+          if [ "$IS_SUCCESS" = "true" ]; then
+            EMAIL_BODY=$(cat <<EOF
+            {
+              "body": "<p>Dear Team,</p><p>We would like to inform you that the ${{ env.accelerator_name }} Test Automation process has completed successfully.</p><p><strong>Run URL:</strong> <a href=\"${RUN_URL}\">${RUN_URL}</a><br></p><p><strong>Test Report:</strong> <a href=\"${REPORT_URL}\">${REPORT_URL}</a></p><p>Best regards,<br>Your Automation Team</p>",
+              "subject": "${{ env.accelerator_name }} Test Automation - Success"
+            }
+          EOF
+            )
+          else
+            EMAIL_BODY=$(cat <<EOF
+            {
+              "body": "<p>Dear Team,</p><p>We would like to inform you that the ${{ env.accelerator_name }} Test Automation process  has encountered an issue and has failed to complete successfully.</p><p><strong>Run URL:</strong> <a href=\"${RUN_URL}\">${RUN_URL}</a><br> ${OUTPUT}</p><p><strong>Test Report:</strong> <a href=\"${REPORT_URL}\">${REPORT_URL}</a></p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>",
+              "subject": "${{ env.accelerator_name }} Test Automation - Failure"
+            }
+          EOF
+            )
+          fi
+
+          # Send the notification
+          curl -X POST "${{ secrets.EMAILNOTIFICATION_LOGICAPP_URL_TA }}" \
+            -H "Content-Type: application/json" \
+            -d "$EMAIL_BODY" || echo "Failed to send notification"

.github/workflows/tests.yml

@@ -78,7 +78,7 @@ jobs:
 
           echo "MIN_COVERAGE=$MIN_COVERAGE" >> "$GITHUB_OUTPUT"
       - name: Run Python Tests
-        run: make python-test optional_args="--junitxml=coverage-junit.xml --cov=. --cov-report xml:coverage.xml --cov-fail-under ${{ steps.coverage-value.outputs.MIN_COVERAGE }}"
+        run: make python-test optional_args="--junitxml=coverage-junit.xml --cov=. --cov-report xml:coverage.xml --cov-fail-under ${{ steps.coverage-value.outputs.MIN_COVERAGE }} ./code/tests"
       - uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
         with:

.gitignore

@@ -418,6 +418,8 @@ temp/
 # so that Azure App Service can install the dependencies
 requirements.txt
 !infra/prompt-flow/cwyd/requirements.txt
+!tests/e2e-test/requirements.txt
+!tests/llm-evaluator/requirements.txt
 
 # Cypress UI tests screenshots folder
 tests/integration/ui/cypress/screenshots/

README.md

@@ -194,10 +194,10 @@ Select either "PostgreSQL" or "Cosmos DB":
 
 When Deployment is complete, follow steps in [Set Up Authentication in Azure App Service](./docs/azure_app_service_auth_setup.md) to add app authentication to your web app running on Azure App Service
 
-**Note**: The default configuration deploys an OpenAI Model "gpt-4o" with version 2024-05-13. However, not all
+**Note**: The default configuration deploys an OpenAI Model "gpt-4.1" with version 2025-04-14. However, not all
 locations support this version. If you're deploying to a location that doesn't support version 2024-05-13, you'll need to
 switch to a lower version. To find out which versions are supported in different regions, visit the
-[GPT-4o Model Availability](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models?tabs=global-standard%2Cstandard-chat-completions#global-standard-model-availability) page.
+[GPT-4.1 Model Availability](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models?tabs=global-standard%2Cstandard-chat-completions#global-standard-model-availability) page.
 
 ### Testing the deployment
 1. Navigate to the admin site, where you can upload documents. It will be located at:

code/backend/batch/utilities/helpers/env_helper.py

@@ -174,10 +174,10 @@ def __load_config(self, **kwargs) -> None:
         else:
             # Otherwise, fallback to individual environment variables
             self.AZURE_OPENAI_MODEL = os.getenv(
-                "AZURE_OPENAI_MODEL", "gpt-4o"
+                "AZURE_OPENAI_MODEL", "gpt-4.1"
             )
             self.AZURE_OPENAI_MODEL_NAME = os.getenv(
-                "AZURE_OPENAI_MODEL_NAME", "gpt-4o"
+                "AZURE_OPENAI_MODEL_NAME", "gpt-4.1"
             )
 
         self.AZURE_OPENAI_VISION_MODEL = os.getenv("AZURE_OPENAI_VISION_MODEL", "gpt-4")

code/backend/pages/01_Ingest_Data.py

@@ -64,6 +64,11 @@ def sanitize_metadata_value(value):
 
 
 def add_url_embeddings(urls: list[str]):
+    has_valid_url = bool(list(filter(str.strip, urls)))
+    if not has_valid_url:
+        st.error("Please enter at least one valid URL.")
+        return
+
     params = {}
     if env_helper.FUNCTION_KEY is not None:
         params["code"] = env_helper.FUNCTION_KEY

code/tests/functional/tests/backend_api/default/test_conversation.py

@@ -65,7 +65,7 @@ def completions_mocking(httpserver: HTTPServer, app_config: AppConfig):
             "id": "chatcmpl-6v7mkQj980V1yBec6ETrKPRqFjNw9",
             "object": "chat.completion",
             "created": 1679072642,
-            "model": "gpt-4o",
+            "model": "gpt-4.1",
             "usage": {
                 "prompt_tokens": 40,
                 "completion_tokens": 50,

code/tests/functional/tests/backend_api/default/test_post_prompt_tool.py

@@ -100,7 +100,7 @@ def completions_mocking(httpserver: HTTPServer, app_config: AppConfig):
             "id": "chatcmpl-6v7mkQj980V1yBec6ETrKPRqFjNw9",
             "object": "chat.completion",
             "created": 1679072642,
-            "model": "gpt-4o",
+            "model": "gpt-4.1",
             "usage": {
                 "prompt_tokens": 40,
                 "completion_tokens": 50,
@@ -132,7 +132,7 @@ def test_post_responds_successfully_when_not_filtered(
             "id": "chatcmpl-6v7mkQj980V1yBec6ETrKPRqFjNw9",
             "object": "chat.completion",
             "created": 1679072642,
-            "model": "gpt-4o",
+            "model": "gpt-4.1",
             "usage": {
                 "prompt_tokens": 40,
                 "completion_tokens": 50,
@@ -193,7 +193,7 @@ def test_post_responds_successfully_when_filtered(
             "id": "chatcmpl-6v7mkQj980V1yBec6ETrKPRqFjNw9",
             "object": "chat.completion",
             "created": 1679072642,
-            "model": "gpt-4o",
+            "model": "gpt-4.1",
             "usage": {
                 "prompt_tokens": 40,
                 "completion_tokens": 50,
@@ -254,7 +254,7 @@ def test_post_makes_correct_call_to_openai_from_post_prompt_tool(
             "id": "chatcmpl-6v7mkQj980V1yBec6ETrKPRqFjNw9",
             "object": "chat.completion",
             "created": 1679072642,
-            "model": "gpt-4o",
+            "model": "gpt-4.1",
             "usage": {
                 "prompt_tokens": 40,
                 "completion_tokens": 50,

code/tests/functional/tests/backend_api/integrated_vectorization_custom_conversation/test_iv_question_answer_tool.py

@@ -63,7 +63,7 @@ def completions_mocking(httpserver: HTTPServer, app_config: AppConfig):
             "id": "chatcmpl-6v7mkQj980V1yBec6ETrKPRqFjNw9",
             "object": "chat.completion",
             "created": 1679072642,
-            "model": "gpt-4o",
+            "model": "gpt-4.1",
             "usage": {
                 "prompt_tokens": 40,
                 "completion_tokens": 50,

docs/LOCAL_DEPLOYMENT.md

@@ -12,6 +12,8 @@ The easiest way to run this accelerator is in a VS Code Dev Containers, which wi
 1. In the VS Code window that opens, once the project files show up (this may take several minutes), open a terminal window
 1. Run `azd auth login`
 1. Run `azd env set AZURE_APP_SERVICE_HOSTING_MODEL code` - This sets your environment to deploy code rather than rely on public containers, like the "Deploy to Azure" button.
+1. To use an existing Log Analytics workspace, follow the [setup steps here](../docs/re-use-log-analytics.md) before running `azd up`.
+1. To use an existing Resource Group, follow the [setup steps here](../docs/re-use-resource-group.md) before running `azd up`.
 1. Run `azd up` - This will provision Azure resources and deploy the accelerator to those resources.
 
     * **Important**: Beware that the resources created by this command will incur immediate costs, primarily from the AI Search resource. These resources may accrue costs even if you interrupt the command before it is fully executed. You can run `azd down` or delete the resources manually to avoid unnecessary spending.
@@ -191,7 +193,7 @@ Execute the above [shell command](#L81) to run the function locally. You may nee
 |AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION ||Whether to use [Integrated Vectorization](https://learn.microsoft.com/en-us/azure/search/vector-search-integrated-vectorization)|
 |AZURE_OPENAI_RESOURCE||the name of your Azure OpenAI resource|
 |AZURE_OPENAI_MODEL||The name of your model deployment|
-|AZURE_OPENAI_MODEL_NAME|gpt-4o|The name of the model|
+|AZURE_OPENAI_MODEL_NAME|gpt-4.1|The name of the model|
 |AZURE_OPENAI_MODEL_VERSION|2024-05-13|The version of the model to use|
 |AZURE_OPENAI_API_KEY||One of the API keys of your Azure OpenAI resource|
 |AZURE_OPENAI_EMBEDDING_MODEL|text-embedding-ada-002|The name of your Azure OpenAI embeddings model deployment|

docs/TEAMS_LOCAL_DEPLOYMENT.md

@@ -65,7 +65,7 @@ Or use the [Azure Functions VS Code extension](https://marketplace.visualstudio.
 |AZURE_SEARCH_FILTER||Filter to apply to search queries.|
 |AZURE_OPENAI_RESOURCE||the name of your Azure OpenAI resource|
 |AZURE_OPENAI_MODEL||The name of your model deployment|
-|AZURE_OPENAI_MODEL_NAME|gpt-4o|The name of the model|
+|AZURE_OPENAI_MODEL_NAME|gpt-4.1|The name of the model|
 |AZURE_OPENAI_API_KEY||One of the API keys of your Azure OpenAI resource|
 |AZURE_OPENAI_EMBEDDING_MODEL|text-embedding-ada-002|The name of you Azure OpenAI embeddings model deployment|
 |AZURE_OPENAI_TEMPERATURE|0|What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. A value of 0 is recommended when using your data.|

docs/azure_app_service_auth_setup.md

@@ -1,58 +1,33 @@
 # Set Up Authentication in Azure App Service
 
+This document provides step-by-step instructions to configure Azure App Registrations for a front-end application.
+
+## Prerequisites
+
+- Access to **Microsoft Entra ID**
+- Necessary permissions to create and manage **App Registrations**
+
 ## Step 1: Add Authentication in Azure App Service configuration
 
 1. Click on `Authentication` from left menu.
 
 ![Authentication](images/AppAuthentication.png)
 
-2. Click on `+ Add Provider` to see a list of identity providers.
+2. Click on `+ Add identity provider` to see a list of identity providers.
 
 ![Authentication Identity](images/AppAuthenticationIdentity.png)
 
-3. Click on `+ Add Provider` to see a list of identity providers.
+3. Click on `Identity Provider` dropdown to see a list of identity providers.
 
 ![Add Provider](images/AppAuthIdentityProvider.png)
 
-4. Select the first option `Microsoft Entra Id` from the drop-down list. If `Create new app registration` is disabled, go to [Step 1a](#step-1a-creating-a-new-app-registration).
+4. Select the first option `Microsoft Entra Id` from the drop-down list and select `client secret expiration` under App registration.
+> NOTE: If `Create new app registration` is disabled, then go to [Create new app registration](/docs/create_new_app_registration.md) and come back to this step to complete the app authentication.
 
 ![Add Provider](images/AppAuthIdentityProviderAdd.png)
 
 5. Accept the default values and click on `Add` button to go back to the previous page with the identify provider added.
 
 ![Add Provider](images/AppAuthIdentityProviderAdded.png)
 
-### Step 1a: Creating a new App Registration
-
-1. Click on `Home` and select `Microsoft Entra ID`.
-
-![Microsoft Entra ID](images/MicrosoftEntraID.png)
-
-2. Click on `App registrations`.
-
-![App registrations](images/Appregistrations.png)
-
-3. Click on `+ New registration`.
-
-![New Registrations](images/NewRegistration.png)
-
-4. Provide the `Name`, select supported account types as `Accounts in this organizational directory only(Contoso only - Single tenant)`, select platform as `Web`, enter/select the `URL` and register.
-
-![Add Details](images/AddDetails.png)
-
-5. After application is created sucessfully, then click on `Add a Redirect URL`.
-
-![Redirect URL](images/AddRedirectURL.png)
-
-6. Click on `+ Add a platform`.
-
-![+ Add platform](images/AddPlatform.png)
-
-7. Click on `Web`.
-
-![Web](images/Web.png)
-
-8. Enter the `web app URL` (Provide the app service name in place of XXXX) and Save. Then go back to [Step 1](#step-1-add-authentication-in-azure-app-service-configuration) and follow from _Point 4_ choose `Pick an existing app registration in this directory` from the Add an Identity Provider page and provide the newly registered App Name.
-E.g. https://appservicename.azurewebsites.net/.auth/login/aad/callback
-
-![Add Details](images/WebAppURL.png)
+6. You have successfully added app authentication, and now required to log in to access the application.

docs/contract_assistance.md

@@ -10,9 +10,9 @@ The following is the Chat With Your Data infrastructure configuration that we su
 - **Azure Semantic Search**: Utilize Azure Semantic Search to efficiently index and search legal documents. This provides powerful search capabilities and integration with other Azure services.
 - **Azure Cognitive Search Top K 15**: Set the Top K parameter to 15 to retrieve the top 15 most relevant documents. This configuration helps in providing precise and relevant search results for user queries.
 - **Azure Search Integrated Vectorization**: Enable integrated vectorization in Azure Search to improve the semantic understanding and relevance of search results. This enhances the Contract Assistant's ability to provide contextually accurate answers.
-- **Azure OpenAI Model gpt-4o**: Leverage the Azure OpenAI model gpt-4o for advanced natural language processing capabilities. This model is well-suited for handling complex legal queries and providing detailed and contextually appropriate responses.
+- **Azure OpenAI Model gpt-4.1**: Leverage the Azure OpenAI model gpt-4.1 for advanced natural language processing capabilities. This model is well-suited for handling complex legal queries and providing detailed and contextually appropriate responses.
 - **Orchestration Strategy: Semantic Kernel**: Implement the Semantic Kernel orchestration strategy to effectively manage the integration and interaction between different components of the infrastructure. This strategy ensures seamless operation and optimal performance of the Contract Assistant.
-- **Conversation Flow Options**: Setting `CONVERSATION_FLOW` enables running advanced AI models like GPT-4o on your own enterprise data without needing to train or fine-tune models.
+- **Conversation Flow Options**: Setting `CONVERSATION_FLOW` enables running advanced AI models like GPT-4.1 on your own enterprise data without needing to train or fine-tune models.
 
 By following these infrastructure configurations, you can enhance the efficiency, accuracy, and overall performance of the Chat With Your Data Contract Review and Summarization Assistant, ensuring it meets the high demands and expectations of  professionals.
 
@@ -22,8 +22,8 @@ To apply the suggested configurations in your deployment, update the following f
 - **Azure Semantic Search**: Set `AZURE_SEARCH_USE_SEMANTIC_SEARCH` to `true`
 - **Azure Cognitive Search Top K 15**: Set `AZURE_SEARCH_TOP_K` to `15`.
 - **Azure Search Integrated Vectorization**: Set `AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION` to `true`.
-- **Azure OpenAI Model**: Set `AZURE_OPENAI_MODEL`  to `gpt-4o`.
-- **Azure OpenAI Model Name**: Set `AZURE_OPENAI_MODEL_NAME` to `gpt-4o`. (could be different based on the name of the Azure OpenAI model deployment)
+- **Azure OpenAI Model**: Set `AZURE_OPENAI_MODEL`  to `gpt-4.1`.
+- **Azure OpenAI Model Name**: Set `AZURE_OPENAI_MODEL_NAME` to `gpt-4.1`. (could be different based on the name of the Azure OpenAI model deployment)
 - **Azure OpenAI Model Name Version**: Set `AZURE_OPENAI_MODEL_VERSION` to `2024-05-13`.
 - **Conversation Flow Options**: Set `CONVERSATION_FLOW` to `byod`
 - **Orchestration Strategy**: Set `ORCHESTRATION_STRATEGY` to `Semantic Kernel`.