Python: Preb1 merge from main 1

.github/_typos.toml

@@ -23,7 +23,10 @@ extend-exclude = [
     "PopulationByCountry.csv",
     "PopulationByAdmin1.csv",
     "WomensSuffrage.txt",
-    "SK-dotnet.sln.DotSettings"
+    "SK-dotnet.sln.DotSettings",
+    "**/azure_ai_search_hotel_samples/README.md",
+    "**/Demos/ProcessFrameworkWithAspire/ProcessFramework.Aspire/ProcessFramework.Aspire.ProcessOrchestrator/Program.cs",
+    "**/Demos/ProcessFrameworkWithAspire/**/*.http"
 ]
 
 [default.extend-words]

.github/workflows/dotnet-build-and-test.yml

@@ -148,11 +148,17 @@ jobs:
 
       # Generate test reports and check coverage
       - name: Generate test reports
-        uses: danielpalme/ReportGenerator-GitHub-Action@5.4.3
+        uses: danielpalme/ReportGenerator-GitHub-Action@5.4.4
         with:
           reports: "./TestResults/Coverage/**/coverage.cobertura.xml"
           targetdir: "./TestResults/Reports"
-          reporttypes: "JsonSummary"
+          reporttypes: "HtmlInline;JsonSummary"
+
+      - name: Upload coverage report artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: CoverageReport-${{ matrix.os }}-${{ matrix.dotnet }}-${{ matrix.configuration }} # Artifact name
+          path: ./TestResults/Reports # Directory containing files to upload
 
       - name: Check coverage
         shell: pwsh

.github/workflows/label-needs-port.yml

@@ -0,0 +1,41 @@
+name: Create Issue when Needs Port label is added
+on:
+  issues:
+    types: [labeled]
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  create_issue:
+    if: contains(github.event.pull_request.labels.*.name, 'needs_port_to_dotnet') || contains(github.event.pull_request.labels.*.name, 'needs_port_to_python') || contains(github.event.issue.labels.*.name, 'needs_port_to_dotnet') || contains(github.event.issue.labels.*.name, 'needs_port_to_python')
+    name: "Create Issue"
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: read
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GH_REPO: ${{ github.repository }}
+
+    steps:
+      - name: Create dotnet issue
+        if: contains(github.event.pull_request.labels.*.name, 'needs_port_to_dotnet') || contains(github.event.issue.labels.*.name, 'needs_port_to_dotnet')
+        run: |
+          new_issue_url=$(gh issue create \
+            --title "Port python feature: ${{ github.event.issue.title || github.event.pull_request.title }}" \
+            --label ".NET" \
+            --body "# Original issue
+            ${{ github.event.issue.html_url || github.event.pull_request.html_url }}
+            ## Description
+            ${{ github.event.issue.body || github.event.pull_request.body }}")
+      - name: Create python issue
+        if: contains(github.event.pull_request.labels.*.name, 'needs_port_to_python') || contains(github.event.issue.labels.*.name, 'needs_port_to_python')
+        run: |
+          new_issue_url=$(gh issue create \
+            --title "Port dotnet feature: ${{ github.event.issue.title || github.event.pull_request.title }}" \
+            --label "python" \
+            --body "# Original issue
+            ${{ github.event.issue.html_url || github.event.pull_request.html_url }}
+            ## Description
+            ${{ github.event.issue.body || github.event.pull_request.body }}")

.github/workflows/python-build.yml

@@ -4,11 +4,16 @@ on:
   release:
     types: [published]
 
+permissions:
+  contents: read
+  id-token: "write"
+
 jobs:
   python-build-assets:
     if: github.event_name == 'release' && startsWith(github.event.release.tag_name, 'python-')
     name: Python Build Assets and add to Release
     runs-on: ubuntu-latest
+    environment: "integration"
     permissions:
       contents: write
     env:
@@ -21,6 +26,7 @@ jobs:
           version: "0.5.x"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
+          cache-dependency-glob: "**/uv.lock"
       - name: Check version
         run: |
           echo "Building and uploading Python package version: ${{ github.event.release.tag_name }}"
@@ -31,3 +37,14 @@ jobs:
         with:
           files: |
             python/dist/*
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      - name: Start DevOps pipeline
+        uses: azure/cli@v2
+        with:
+          inlineScript: |
+            az pipelines run --id ${{ vars.ADO_PYTHON_RELEASE_ID }} --org ${{ vars.ADO_ORG }} --project ${{ vars.ADO_PROJECT_NAME }} --parameters tag=${{ github.event.release.tag_name }} delay=0

.github/workflows/python-integration-tests.yml

@@ -55,10 +55,10 @@ env:
   MISTRALAI_EMBEDDING_MODEL_ID: ${{ vars.MISTRALAI_EMBEDDING_MODEL_ID }}
   ANTHROPIC_API_KEY: ${{secrets.ANTHROPIC_API_KEY}}
   ANTHROPIC_CHAT_MODEL_ID: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }}
-  OLLAMA_CHAT_MODEL_ID: "${{ vars.OLLAMA_CHAT_MODEL_ID || '' }}" # llava-phi3
-  OLLAMA_CHAT_MODEL_ID_IMAGE: "${{ vars.OLLAMA_CHAT_MODEL_ID_IMAGE || '' }}" # llava-phi3
-  OLLAMA_CHAT_MODEL_ID_TOOL_CALL: "${{ vars.OLLAMA_CHAT_MODEL_ID_TOOL_CALL || '' }}" # llama3.2
-  OLLAMA_TEXT_MODEL_ID: "${{ vars.OLLAMA_TEXT_MODEL_ID || '' }}" # llava-phi3
+  OLLAMA_CHAT_MODEL_ID: "${{ vars.OLLAMA_CHAT_MODEL_ID || '' }}" # llama3.2:1b
+  OLLAMA_CHAT_MODEL_ID_IMAGE: "${{ vars.OLLAMA_CHAT_MODEL_ID_IMAGE || '' }}" # moondream
+  OLLAMA_CHAT_MODEL_ID_TOOL_CALL: "${{ vars.OLLAMA_CHAT_MODEL_ID_TOOL_CALL || '' }}" # llama3.2:1b
+  OLLAMA_TEXT_MODEL_ID: "${{ vars.OLLAMA_TEXT_MODEL_ID || '' }}" # llama3.2:1b
   OLLAMA_EMBEDDING_MODEL_ID: "${{ vars.OLLAMA_EMBEDDING_MODEL_ID || '' }}" # nomic-embed-text
   GOOGLE_AI_GEMINI_MODEL_ID: ${{ vars.GOOGLE_AI_GEMINI_MODEL_ID }}
   GOOGLE_AI_EMBEDDING_MODEL_ID: ${{ vars.GOOGLE_AI_EMBEDDING_MODEL_ID }}
@@ -69,6 +69,8 @@ env:
   REDIS_CONNECTION_STRING: ${{ vars.REDIS_CONNECTION_STRING }}
   AZURE_COSMOS_DB_NO_SQL_URL: ${{ vars.AZURE_COSMOS_DB_NO_SQL_URL }}
   AZURE_COSMOS_DB_NO_SQL_KEY: ${{ secrets.AZURE_COSMOS_DB_NO_SQL_KEY }}
+  BEDROCK_AGENT_AGENT_RESOURCE_ROLE_ARN: ${{ secrets.BEDROCK_AGENT_AGENT_RESOURCE_ROLE_ARN }}
+  BEDROCK_AGENT_FOUNDATION_MODEL: ${{ vars.BEDROCK_AGENT_FOUNDATION_MODEL }}
 
 jobs:
   paths-filter:
@@ -118,6 +120,7 @@ jobs:
           version: "0.5.x"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
+          cache-dependency-glob: "**/uv.lock"
       - name: Install dependencies
         run: |
           uv sync --all-extras --dev
@@ -147,10 +150,92 @@ jobs:
         run: |
           uv run pytest -v --log-cli-level=INFO --durations=20 -n logical --dist loadfile --dist worksteal -m "not ollama" ./tests/integration/completions ./tests/integration/embeddings ./tests/samples ./tests/integration/cross_language
 
+  python-merge-gate-multi-modality:
+    name: Python Pre-Merge Integration Tests - Multi-Modality
+    needs: paths-filter
+    if: github.event_name != 'pull_request' && github.event_name != 'schedule' && needs.paths-filter.outputs.pythonChanges == 'true'
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"]
+        os: [ubuntu-latest]
+    defaults:
+      run:
+        working-directory: python
+    runs-on: ${{ matrix.os }}
+    environment: "integration"
+    env:
+      UV_PYTHON: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+          enable-cache: true
+          cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          uv sync --all-extras --dev
+      - name: Azure CLI Login
+        if: github.event_name != 'pull_request'
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      - name: Run Integration Tests
+        id: run_tests_multi_modality
+        shell: bash
+        run: |
+          uv run pytest -v --log-cli-level=INFO --durations=20 -n logical --dist loadfile --dist worksteal ./tests/integration/audio_to_text ./tests/integration/text_to_audio ./tests/integration/text_to_image
+
+  python-merge-gate-agents:
+    name: Python Pre-Merge Integration Tests - Agents
+    needs: paths-filter
+    if: github.event_name != 'pull_request' && github.event_name != 'schedule' && needs.paths-filter.outputs.pythonChanges == 'true'
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+      matrix:
+        python-version: ["3.11"]
+        os: [ubuntu-latest]
+    defaults:
+      run:
+        working-directory: python
+    runs-on: ${{ matrix.os }}
+    environment: "integration"
+    env:
+      UV_PYTHON: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+          enable-cache: true
+          cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          uv sync --all-extras --dev
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_REGION }}
+      - name: Run Integration Tests
+        id: run_tests_agents
+        shell: bash
+        run: |
+          uv run pytest -v --log-cli-level=INFO --durations=20 -n logical --dist loadfile --dist worksteal ./tests/integration/agents
+
   python-merge-gate-ollama:
     name: Python Pre-Merge Integration Tests - Ollama
     needs: paths-filter
-    if: github.event_name != 'pull_request' && github.event_name != 'schedule' && needs.paths-filter.outputs.pythonChanges == 'true'
+    # Ollama tests are very unstable at the moment. It often fails to pull models from the Ollama server. Thus, this job is disabled for now.
+    if: false && github.event_name != 'pull_request' && github.event_name != 'schedule' && needs.paths-filter.outputs.pythonChanges == 'true'
     strategy:
       max-parallel: 1
       fail-fast: false
@@ -173,6 +258,7 @@ jobs:
           version: "0.5.x"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
+          cache-dependency-glob: "**/uv.lock"
       - name: Install dependencies
         run: |
           uv sync --all-extras --dev
@@ -237,6 +323,7 @@ jobs:
           version: "0.5.x"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
+          cache-dependency-glob: "**/uv.lock"
       - name: Install dependencies
         run: |
           uv sync --all-extras --dev
@@ -268,7 +355,6 @@ jobs:
       run:
         working-directory: python
     runs-on: ${{ matrix.os }}
-    environment: "integration"
     env:
       UV_PYTHON: ${{ matrix.python-version }}
       MEMORY_CONCEPT_SAMPLE: "true"
@@ -295,6 +381,7 @@ jobs:
           version: "0.5.x"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
+          cache-dependency-glob: "**/uv.lock"
       - name: Install dependencies
         run: |
           uv sync --all-extras --dev
@@ -305,7 +392,8 @@ jobs:
           ollama serve &
           sleep 5
       - name: Pull model in Ollama
-        if: matrix.os == 'ubuntu-latest'
+        # Ollama tests are very unstable at the moment. It often fails to pull models from the Ollama server. Thus, Ollama is disabled for now.
+        if: false && matrix.os == 'ubuntu-latest'
         run: |
           ollama pull ${{ vars.OLLAMA_CHAT_MODEL_ID }}
           ollama pull ${{ vars.OLLAMA_CHAT_MODEL_ID_IMAGE }}
@@ -343,14 +431,16 @@ jobs:
         id: run_tests_completions
         timeout-minutes: 10
         shell: bash
+        # Ollama tests are very unstable at the moment. It often fails to pull models from the Ollama server. Thus, Ollama is disabled for now.
         run: |
-          uv run pytest -v -n logical --dist loadfile --dist worksteal ./tests/integration/completions
+          uv run pytest -v -n logical --dist loadfile --dist worksteal -m "not ollama" ./tests/integration/completions
       - name: Run Integration Tests - Embeddings
         id: run_tests_embeddings
         timeout-minutes: 5
         shell: bash
+        # Ollama tests are very unstable at the moment. It often fails to pull models from the Ollama server. Thus, Ollama is disabled for now.
         run: |
-          uv run pytest -v -n logical --dist loadfile --dist worksteal ./tests/integration/embeddings
+          uv run pytest -v -n logical --dist loadfile --dist worksteal -m "not ollama" ./tests/integration/embeddings
       - name: Run Integration Tests - Memory
         id: run_tests_memory
         timeout-minutes: 5
@@ -375,6 +465,18 @@ jobs:
         shell: bash
         run: |
           uv run pytest -v -n logical --dist loadfile --dist worksteal ./tests/samples
+      - name: Run Integration Tests - Agents
+        id: run_tests_agents
+        timeout-minutes: 5
+        shell: bash
+        run: |
+          uv run pytest -v -n logical --dist loadfile --dist worksteal ./tests/integration/agents
+      - name: Run Integration Tests - Multi-Modality
+        id: run_tests_multi_modality
+        timeout-minutes: 5
+        shell: bash
+        run: |
+          uv run pytest -v -n logical --dist loadfile --dist worksteal ./tests/integration/audio_to_text ./tests/integration/text_to_audio ./tests/integration/text_to_image
 
   # This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed
   python-integration-tests-check:
@@ -388,6 +490,8 @@ jobs:
         python-merge-gate-ai-services,
         python-merge-gate-ollama,
         python-merge-gate-memory,
+        python-merge-gate-agents,
+        python-merge-gate-multi-modality,
         python-integration-tests,
       ]
     steps:

.github/workflows/python-lint.yml

@@ -30,6 +30,7 @@ jobs:
           version: "0.5.x"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ matrix.python-version }}
+          cache-dependency-glob: "**/uv.lock"
       - name: Install the project
         run: uv sync --all-extras --dev
       - uses: pre-commit/action@v3.0.1

.github/workflows/python-manual-release.yml

@@ -0,0 +1,30 @@
+name: Python Start Release on ADO
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Tag to release"
+        required: true
+
+permissions:
+  contents: read
+  id-token: "write"
+
+jobs:
+  python-build-assets:
+    name: Trigger ADO Pipeline for Python Release
+    runs-on: ubuntu-latest
+    environment: "integration"
+    steps:
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      - name: Start DevOps pipeline
+        uses: azure/cli@v2
+        with:
+          inlineScript: |
+            az pipelines run --id ${{ vars.ADO_PYTHON_RELEASE_ID }} --org ${{ vars.ADO_ORG }} --project ${{ vars.ADO_PROJECT_NAME }}  --parameters tag=${{ inputs.tag }} delay=0

.github/workflows/python-test-coverage.yml

@@ -32,6 +32,7 @@ jobs:
           version: "0.5.x"
           enable-cache: true
           cache-suffix: ${{ runner.os }}-${{ env.UV_PYTHON }}
+          cache-dependency-glob: "**/uv.lock"
       - name: Install the project
         run: uv sync --all-extras --dev
       - name: Test with pytest