diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index c96338a2..066b841d 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -27,73 +27,109 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest-xlarge", "macos-13"] + os: ["ubuntu-latest", "macos-latest-xlarge", "macos-13"] python_version: ["3.10", "3.11", "3.12"] exclude: - os: macos-latest-xlarge python_version: "3.10" + env: + UV_SYSTEM_PYTHON: 1 + POSTGRES_HOST: localhost + POSTGRES_USERNAME: postgres + POSTGRES_PASSWORD: root + POSTGRES_DATABASE: postgres + POSTGRES_SSL: disable steps: - uses: actions/checkout@v4 - - name: Check for MacOS Runner - if: matrix.os == 'macos-latest-xlarge' - run: brew install postgresql@14 - - name: Install pgvector on Windows using install-pgvector.bat + + - name: (MacOS) Install postgreSQL and pgvector using brew + if: matrix.os == 'macos-13' || matrix.os == 'macos-latest-xlarge' + run: | + brew install postgresql@14 + brew link --overwrite postgresql@14 + brew install pgvector + brew services start postgresql@14 && sleep 1 + createuser -s ${{ env.POSTGRES_USERNAME }} + psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'" + psql -d postgres -c 'CREATE EXTENSION vector' + + - name: (Windows) Install pgvector using install-pgvector.bat if: matrix.os == 'windows-latest' shell: cmd run: .github\workflows\install-pgvector.bat - - name: Install PostgreSQL development libraries - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update - sudo apt install postgresql-server-dev-14 - - name: Setup postgres + + - name: (Windows) Start postgreSQL + if: matrix.os == 'windows-latest' uses: ikalnytskyi/action-setup-postgres@v6 with: - username: admin - password: postgres - database: postgres - - name: Install pgvector on MacOS/Linux using install-pgvector.sh - if: matrix.os != 'windows-latest' - run: .github/workflows/install-pgvector.sh + username: ${{ env.POSTGRES_USERNAME }} + password: ${{ env.POSTGRES_PASSWORD }} + database: ${{ env.POSTGRES_DATABASE }} + + - name: (Linux) Install pgvector and set password + if: matrix.os == 'ubuntu-latest' + run: | + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get install postgresql-14-pgvector + sudo systemctl start postgresql + sudo -u postgres psql -c "ALTER USER ${{ env.POSTGRES_USERNAME }} PASSWORD '${{ env.POSTGRES_PASSWORD }}'" + sudo -u postgres psql -c 'CREATE EXTENSION vector' + - name: Setup python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} architecture: x64 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + version: "0.4.20" + cache-dependency-glob: "requirements**.txt" + - name: Install dependencies run: | - python -m pip install -r requirements-dev.txt + uv pip install -r requirements-dev.txt + - name: Install app as editable app run: | - python -m pip install -e src/backend + uv pip install -e src/backend + - name: Setup local database with seed data run: | - cp .env.sample .env python ./src/backend/fastapi_app/setup_postgres_database.py python ./src/backend/fastapi_app/setup_postgres_seeddata.py + - name: Setup node uses: actions/setup-node@v4 with: node-version: 18 + - name: Build frontend run: | cd ./src/frontend npm install npm run build - - name: cache mypy + + - name: Setup mypy cache uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 with: path: ./.mypy_cache key: mypy${{ matrix.os }}-${{ matrix.python_version }}-${{ hashFiles('requirements-dev.txt', 'src/backend/requirements.txt', 'src/backend/pyproject.toml') }} + - name: Run MyPy run: python3 -m mypy . + - name: Run Pytest run: python3 -m pytest -s -vv --cov --cov-fail-under=85 + - name: Run E2E tests with Playwright id: e2e run: | playwright install chromium --with-deps python3 -m pytest tests/e2e.py --tracing=retain-on-failure + - name: Upload test artifacts if: ${{ failure() && steps.e2e.conclusion == 'failure' }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 73220be3..1452cd11 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -1,7 +1,6 @@ name: Evaluate on: - workflow_dispatch: issue_comment: types: [created] @@ -13,14 +12,32 @@ permissions: jobs: evaluate: - if: github.event_name == 'workflow_dispatch' || contains(github.event.comment.body, '#evaluate') + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '#evaluate') runs-on: ubuntu-latest env: + UV_SYSTEM_PYTHON: 1 AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} + POSTGRES_HOST: localhost + POSTGRES_USERNAME: postgres + POSTGRES_PASSWORD: root + POSTGRES_DATABASE: postgres + POSTGRES_SSL: disable + OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} + OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} + AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} + AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} + AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }} + AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }} + AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }} + AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} + AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }} + AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }} steps: - name: Check for evaluate hash tag if: contains(github.event.comment.body, '#evaluate') @@ -28,62 +45,35 @@ jobs: echo "Comment contains #evaluate hashtag" - uses: actions/checkout@v4 - - name: Install PostgreSQL development libraries + + - name: Install pgvector run: | - sudo apt update - sudo apt install postgresql-server-dev-14 - - name: Setup postgres - uses: ikalnytskyi/action-setup-postgres@v6 - with: - username: admin - password: postgres - database: postgres + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get install postgresql-14-pgvector + + - name: Start postgres + run: sudo systemctl start postgresql + + - name: Set password for postgres user + run: sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'root'" - - name: Install pgvector on MacOS/Linux using install-pgvector.sh - run: .github/workflows/install-pgvector.sh + - name: Create vector extension + run: sudo -u postgres psql -c 'CREATE EXTENSION vector' - name: Install python uses: actions/setup-python@v5 with: python-version: '3.12' - - name: Install azd - uses: Azure/setup-azd@v1.0.0 - - - name: Install dependencies - run: | - python -m pip install -r requirements-dev.txt - - - name: Install app as editable app - run: | - python -m pip install -e src/backend - - - name: Setup local database with seed data - run: | - python ./src/backend/fastapi_app/setup_postgres_database.py - python ./src/backend/fastapi_app/setup_postgres_seeddata.py - env: - POSTGRES_HOST: localhost - POSTGRES_USERNAME: admin - POSTGRES_PASSWORD: postgres - POSTGRES_DATABASE: postgres - POSTGRES_SSL: disable - - - name: Setup node - uses: actions/setup-node@v4 + - name: Install uv + uses: astral-sh/setup-uv@v3 with: - node-version: 18 + enable-cache: true + version: "0.4.20" + cache-dependency-glob: "requirements**.txt" - - name: Build frontend - run: | - cd ./src/frontend - npm install - npm run build - - - name: Install python packages - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt + - name: Install azd + uses: Azure/setup-azd@v1.0.0 - name: Login to Azure uses: azure/login@v2 @@ -107,41 +97,42 @@ jobs: --tenant-id "$Env:AZURE_TENANT_ID" shell: pwsh - - name: Provision Infrastructure - run: azd provision --no-prompt - env: - AZD_INITIAL_ENVIRONMENT_CONFIG: ${{ secrets.AZD_INITIAL_ENVIRONMENT_CONFIG }} + - name: Install dependencies + run: | + uv pip install -r requirements-dev.txt + + - name: Install app as editable app + run: | + uv pip install -e src/backend + + - name: Setup local database with seed data + run: | + python ./src/backend/fastapi_app/setup_postgres_database.py + python ./src/backend/fastapi_app/setup_postgres_seeddata.py + + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Build frontend + run: | + cd ./src/frontend + npm install + npm run build - name: Run local server in background run: | RUNNER_TRACKING_ID="" && (nohup python3 -m uvicorn fastapi_app:create_app --factory > serverlogs.out 2> serverlogs.err &) - env: - OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} - OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }} - AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} - AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} - AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} - AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }} - AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }} - AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }} - AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} - POSTGRES_HOST: localhost - POSTGRES_USERNAME: admin - POSTGRES_PASSWORD: postgres - POSTGRES_DATABASE: postgres - POSTGRES_SSL: disable + + - name: Install evaluate dependencies + run: | + uv pip install -r evals/requirements.txt + - name: Evaluate local RAG flow run: | - python evals/evaluate.py - env: - OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} - AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} - AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} - AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} - AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }} - AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }} + python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=results/pr${{ github.event.issue.number }} + - name: Upload server logs as build artifact uses: actions/upload-artifact@v4 with: @@ -158,13 +149,10 @@ jobs: uses: actions/upload-artifact@v4 with: name: eval_result - path: ./src/api/evaluate/eval_results.jsonl + path: ./evals/results/pr${{ github.event.issue.number }} - name: GitHub Summary Step if: ${{ success() }} - working-directory: ./src/api run: | - echo "" >> $GITHUB_STEP_SUMMARY - - echo "📊 Promptflow Evaluation Results" >> $GITHUB_STEP_SUMMARY - cat evaluate/eval_results.md >> $GITHUB_STEP_SUMMARY + echo "📊 Evaluation Results" >> $GITHUB_STEP_SUMMARY + python -m evaltools summary evals/results --output=markdown >> $GITHUB_STEP_SUMMARY diff --git a/README.md b/README.md index 43069b89..6ede71d3 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ Since the local app uses OpenAI models, you should first deploy it for the optim ``` 3. To use OpenAI.com OpenAI, set `OPENAI_CHAT_HOST` and `OPENAI_EMBED_HOST` to "openai". Then fill in the value for `OPENAICOM_KEY`. -4. To use Ollama, set `OPENAI_CHAT_HOST` to "ollama". Then update the values for `OLLAMA_ENDPOINT` and `OLLAMA_CHAT_MODEL` to match your local setup and model. Note that most Ollama models are not compatible with the "Advanced flow", due to the need for function calling support, so you'll need to disable that in _Developer Settings_ in the UI. In addition, the database rows are embedded using the default OpenAI embedding model, so you can't search them using an Ollama embedding model. You can either choose to set `OPENAI_EMBED_HOST` to "azure" or "openai", or turn off vector search in _Developer Settings_. +4. To use Ollama, set `OPENAI_CHAT_HOST` to "ollama". Then update the values for `OLLAMA_ENDPOINT` and `OLLAMA_CHAT_MODEL` to match your local setup and model. We recommend using "llama3.1" for the chat model, since it has support for function calling, and "nomic-embed-text" for the embedding model, since the sample data has already been embedded with this model. If you cannot use function calling, then turn off "Advanced flow" in the Developer Settings. If you cannot use the embedding model, then turn off vector search in the Developer Settings. ### Running the frontend and backend diff --git a/evals/evaluate.py b/evals/evaluate.py index 812a3f09..9ec2eb31 100644 --- a/evals/evaluate.py +++ b/evals/evaluate.py @@ -1,3 +1,4 @@ +import argparse import logging import os from pathlib import Path @@ -50,11 +51,20 @@ def get_openai_config() -> dict: ) load_dotenv(".env", override=True) + parser = argparse.ArgumentParser(description="Run evaluation with OpenAI configuration.") + parser.add_argument("--targeturl", type=str, help="Specify the target URL.") + parser.add_argument("--resultsdir", type=Path, help="Specify the results directory.") + parser.add_argument("--numquestions", type=int, help="Specify the number of questions.") + + args = parser.parse_args() + openai_config = get_openai_config() - # TODO: specify the localhost URL using argument - # TODO: specify the experiment name (based on PR number) - # TODO: Specify the num questions using argument run_evaluate_from_config( - working_dir=Path(__file__).parent, config_path="eval_config.json", openai_config=openai_config, num_questions=2 + working_dir=Path(__file__).parent, + config_path="eval_config.json", + num_questions=args.numquestions, + target_url=args.targeturl, + results_dir=args.resultsdir, + openai_config=openai_config, ) diff --git a/evals/requirements.txt b/evals/requirements.txt new file mode 100644 index 00000000..7559267e --- /dev/null +++ b/evals/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/Azure-Samples/ai-rag-chat-evaluator/@installable diff --git a/infra/main.bicep b/infra/main.bicep index e55cd5ec..8d5ce872 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -449,7 +449,7 @@ output AZURE_OPENAI_RESOURCE_GROUP string = deployAzureOpenAI ? openAIResourceGr output AZURE_OPENAI_ENDPOINT string = !empty(azureOpenAIEndpoint) ? azureOpenAIEndpoint : (deployAzureOpenAI ? openAI.outputs.endpoint : '') -output AZURE_OPENAI_VERSION string = openAIEmbedHost == 'chat' ? azureOpenAIAPIVersion : '' +output AZURE_OPENAI_VERSION string = azureOpenAIAPIVersion output AZURE_OPENAI_CHAT_DEPLOYMENT string = deployAzureOpenAI ? chatDeploymentName : '' output AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION string = deployAzureOpenAI ? chatDeploymentVersion : '' output AZURE_OPENAI_CHAT_DEPLOYMENT_CAPACITY int = deployAzureOpenAI ? chatDeploymentCapacity : 0 diff --git a/requirements-dev.txt b/requirements-dev.txt index cc2180eb..7e6082fa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ -r src/backend/requirements.txt +-r evals/requirements.txt ruff mypy types-requests @@ -12,7 +13,6 @@ pytest-cov pytest-playwright pytest-snapshot locust -git+https://github.com/Azure-Samples/ai-rag-chat-evaluator/@installable psycopg2 azure-mgmt-cognitiveservices dotenv-azd diff --git a/src/backend/fastapi_app/api_models.py b/src/backend/fastapi_app/api_models.py index e1c71b93..35068a80 100644 --- a/src/backend/fastapi_app/api_models.py +++ b/src/backend/fastapi_app/api_models.py @@ -80,6 +80,10 @@ class ItemPublic(BaseModel): class ItemWithDistance(ItemPublic): distance: float + def __init__(self, **data): + super().__init__(**data) + self.distance = round(self.distance, 2) + class ChatParams(ChatRequestOverrides): prompt_template: str diff --git a/src/backend/fastapi_app/dependencies.py b/src/backend/fastapi_app/dependencies.py index d3d0177c..5876c1eb 100644 --- a/src/backend/fastapi_app/dependencies.py +++ b/src/backend/fastapi_app/dependencies.py @@ -37,34 +37,35 @@ class FastAPIAppContext(BaseModel): async def common_parameters(): """ Get the common parameters for the FastAPI app + Use the pattern of `os.getenv("VAR_NAME") or "default_value"` to avoid empty string values """ OPENAI_EMBED_HOST = os.getenv("OPENAI_EMBED_HOST") OPENAI_CHAT_HOST = os.getenv("OPENAI_CHAT_HOST") if OPENAI_EMBED_HOST == "azure": - openai_embed_deployment = os.getenv("AZURE_OPENAI_EMBED_DEPLOYMENT", "text-embedding-ada-002") - openai_embed_model = os.getenv("AZURE_OPENAI_EMBED_MODEL", "text-embedding-ada-002") - openai_embed_dimensions = int(os.getenv("AZURE_OPENAI_EMBED_DIMENSIONS", 1536)) - embedding_column = os.getenv("AZURE_OPENAI_EMBEDDING_COLUMN", "embedding_ada002") + openai_embed_deployment = os.getenv("AZURE_OPENAI_EMBED_DEPLOYMENT") or "text-embedding-ada-002" + openai_embed_model = os.getenv("AZURE_OPENAI_EMBED_MODEL") or "text-embedding-ada-002" + openai_embed_dimensions = int(os.getenv("AZURE_OPENAI_EMBED_DIMENSIONS") or 1536) + embedding_column = os.getenv("AZURE_OPENAI_EMBEDDING_COLUMN") or "embedding_ada002" elif OPENAI_EMBED_HOST == "ollama": openai_embed_deployment = None - openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL", "nomic-embed-text") + openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL") or "nomic-embed-text" openai_embed_dimensions = None - embedding_column = os.getenv("OLLAMA_EMBEDDING_COLUMN", "embedding_nomic") + embedding_column = os.getenv("OLLAMA_EMBEDDING_COLUMN") or "embedding_nomic" else: openai_embed_deployment = None - openai_embed_model = os.getenv("OPENAICOM_EMBED_MODEL", "text-embedding-ada-002") + openai_embed_model = os.getenv("OPENAICOM_EMBED_MODEL") or "text-embedding-ada-002" openai_embed_dimensions = int(os.getenv("OPENAICOM_EMBED_DIMENSIONS", 1536)) - embedding_column = os.getenv("OPENAICOM_EMBEDDING_COLUMN", "embedding_ada002") + embedding_column = os.getenv("OPENAICOM_EMBEDDING_COLUMN") or "embedding_ada002" if OPENAI_CHAT_HOST == "azure": - openai_chat_deployment = os.getenv("AZURE_OPENAI_CHAT_DEPLOYMENT", "gpt-4o-mini") - openai_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL", "gpt-4o-mini") + openai_chat_deployment = os.getenv("AZURE_OPENAI_CHAT_DEPLOYMENT") or "gpt-4o-mini" + openai_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL") or "gpt-4o-mini" elif OPENAI_CHAT_HOST == "ollama": openai_chat_deployment = None - openai_chat_model = os.getenv("OLLAMA_CHAT_MODEL", "phi3:3.8b") - openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL", "nomic-embed-text") + openai_chat_model = os.getenv("OLLAMA_CHAT_MODEL") or "phi3:3.8b" + openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL") or "nomic-embed-text" else: openai_chat_deployment = None - openai_chat_model = os.getenv("OPENAICOM_CHAT_MODEL", "gpt-3.5-turbo") + openai_chat_model = os.getenv("OPENAICOM_CHAT_MODEL") or "gpt-3.5-turbo" return FastAPIAppContext( openai_chat_model=openai_chat_model, openai_embed_model=openai_embed_model, diff --git a/src/backend/fastapi_app/openai_clients.py b/src/backend/fastapi_app/openai_clients.py index 0610aece..10d2f506 100644 --- a/src/backend/fastapi_app/openai_clients.py +++ b/src/backend/fastapi_app/openai_clients.py @@ -62,7 +62,7 @@ async def create_openai_embed_client( openai_embed_client: openai.AsyncAzureOpenAI | openai.AsyncOpenAI OPENAI_EMBED_HOST = os.getenv("OPENAI_EMBED_HOST") if OPENAI_EMBED_HOST == "azure": - api_version = os.environ["AZURE_OPENAI_VERSION"] + api_version = os.environ["AZURE_OPENAI_VERSION"] or "2024-03-01-preview" azure_endpoint = os.environ["AZURE_OPENAI_ENDPOINT"] azure_deployment = os.environ["AZURE_OPENAI_EMBED_DEPLOYMENT"] if api_key := os.getenv("AZURE_OPENAI_KEY"): diff --git a/src/backend/fastapi_app/postgres_engine.py b/src/backend/fastapi_app/postgres_engine.py index 364dc3e9..1b91a10e 100644 --- a/src/backend/fastapi_app/postgres_engine.py +++ b/src/backend/fastapi_app/postgres_engine.py @@ -2,7 +2,9 @@ import os from azure.identity import AzureDeveloperCliCredential +from pgvector.asyncpg import register_vector from sqlalchemy import event +from sqlalchemy.engine import AdaptedConnection from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine from fastapi_app.dependencies import get_azure_credential @@ -30,10 +32,12 @@ def get_password_from_azure_credential(): if sslmode: DATABASE_URI += f"?ssl={sslmode}" - engine = create_async_engine( - DATABASE_URI, - echo=False, - ) + engine = create_async_engine(DATABASE_URI, echo=False) + + @event.listens_for(engine.sync_engine, "connect") + def register_custom_types(dbapi_connection: AdaptedConnection, *args): + logger.info("Registering pgvector extension...") + dbapi_connection.run_async(register_vector) @event.listens_for(engine.sync_engine, "do_connect") def update_password_token(dialect, conn_rec, cargs, cparams): diff --git a/src/backend/fastapi_app/postgres_searcher.py b/src/backend/fastapi_app/postgres_searcher.py index 1953d2a1..82dd5b63 100644 --- a/src/backend/fastapi_app/postgres_searcher.py +++ b/src/backend/fastapi_app/postgres_searcher.py @@ -1,5 +1,5 @@ +import numpy as np from openai import AsyncAzureOpenAI, AsyncOpenAI -from pgvector.utils import to_db from sqlalchemy import Float, Integer, column, select, text from sqlalchemy.ext.asyncio import AsyncSession @@ -87,7 +87,7 @@ async def search( results = ( await self.db_session.execute( sql, - {"embedding": to_db(query_vector), "query": query_text, "k": 60}, + {"embedding": np.array(query_vector), "query": query_text, "k": 60}, ) ).fetchall() diff --git a/src/backend/fastapi_app/prompts/query.txt b/src/backend/fastapi_app/prompts/query.txt index 672b03be..6bbb0a23 100644 --- a/src/backend/fastapi_app/prompts/query.txt +++ b/src/backend/fastapi_app/prompts/query.txt @@ -3,4 +3,4 @@ You have access to an Azure PostgreSQL database with an items table that has col Generate a search query based on the conversation and the new question. If the question is not in English, translate the question to English before generating the search query. If you cannot generate a search query, return the original user question. -DO NOT return anything besides the query. \ No newline at end of file +DO NOT return anything besides the query. diff --git a/src/backend/fastapi_app/routes/api_routes.py b/src/backend/fastapi_app/routes/api_routes.py index 57cbe3a1..8dc1d8bd 100644 --- a/src/backend/fastapi_app/routes/api_routes.py +++ b/src/backend/fastapi_app/routes/api_routes.py @@ -5,7 +5,7 @@ import fastapi from fastapi import HTTPException from fastapi.responses import StreamingResponse -from sqlalchemy import select +from sqlalchemy import select, text from fastapi_app.api_models import ( ChatRequest, @@ -54,15 +54,18 @@ async def similar_handler( if not item: raise HTTPException(detail=f"Item with ID {id} not found.", status_code=404) - closest = await database_session.execute( - select(Item, Item.embedding_ada002.l2_distance(item.embedding_ada002)) - .filter(Item.id != id) - .order_by(Item.embedding_ada002.l2_distance(item.embedding_ada002)) - .limit(n) - ) - return [ - ItemWithDistance.model_validate(item.to_dict() | {"distance": round(distance, 2)}) for item, distance in closest - ] + closest = ( + await database_session.execute( + text( + f"SELECT *, {context.embedding_column} <=> :embedding as DISTANCE FROM {Item.__tablename__} " + "WHERE id <> :item_id ORDER BY distance LIMIT :n" + ), + {"embedding": item.embedding_ada002, "n": n, "item_id": id}, + ) + ).fetchall() + + items = [dict(row._mapping) for row in closest] + return [ItemWithDistance.model_validate(item) for item in items] @router.get("/search", response_model=list[ItemPublic]) diff --git a/src/backend/fastapi_app/setup_postgres_seeddata.py b/src/backend/fastapi_app/setup_postgres_seeddata.py index 28956c58..05a37cf8 100644 --- a/src/backend/fastapi_app/setup_postgres_seeddata.py +++ b/src/backend/fastapi_app/setup_postgres_seeddata.py @@ -4,6 +4,7 @@ import logging import os +import numpy as np import sqlalchemy.exc from dotenv import load_dotenv from sqlalchemy import select, text @@ -41,8 +42,11 @@ async def seed_data(engine): if db_item.scalars().first(): continue attrs = {key: value for key, value in seed_data_object.items()} - row = Item(**attrs) - session.add(row) + attrs["embedding_ada002"] = np.array(seed_data_object["embedding_ada002"]) + attrs["embedding_nomic"] = np.array(seed_data_object["embedding_nomic"]) + column_names = ", ".join(attrs.keys()) + values = ", ".join([f":{key}" for key in attrs.keys()]) + await session.execute(text(f"INSERT INTO {table_name} ({column_names}) VALUES ({values})"), attrs) try: await session.commit() except sqlalchemy.exc.IntegrityError: diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index d697f4d7..acaca112 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "aiohttp>=3.9.5,<4.0.0", "asyncpg>=0.29.0,<1.0.0", "SQLAlchemy[asyncio]>=2.0.30,<3.0.0", - "pgvector>=0.2.5,<0.3.0", + "pgvector>=0.3.0,<0.4.0", "openai>=1.34.0,<2.0.0", "tiktoken>=0.7.0,<0.8.0", "openai-messages-token-helper>=0.1.8,<0.2.0", diff --git a/tests/conftest.py b/tests/conftest.py index 87a1eb22..e7665237 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,10 +24,11 @@ from tests.data import test_data from tests.mocks import MockAzureCredential +# Always use localhost for testing POSTGRES_HOST = "localhost" -POSTGRES_USERNAME = "admin" -POSTGRES_DATABASE = "postgres" -POSTGRES_PASSWORD = "postgres" +POSTGRES_USERNAME = os.getenv("POSTGRES_USERNAME", "admin") +POSTGRES_DATABASE = os.getenv("POSTGRES_DATABASE", "postgres") +POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD", "postgres") POSTGRES_SSL = "prefer" POSTGRESQL_DATABASE_URL = ( f"postgresql+asyncpg://{POSTGRES_USERNAME}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}/{POSTGRES_DATABASE}" diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json index 7ad46375..d9f9762d 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json @@ -20,7 +20,7 @@ "description": [ { "role": "system", - "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\nGenerate a search query based on the conversation and the new question.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return the original user question.\nDO NOT return anything besides the query." + "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\nGenerate a search query based on the conversation and the new question.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return the original user question.\nDO NOT return anything besides the query.\n" }, { "role": "user", diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines index 3615937f..9f5aaa63 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":[{"role":"system","content":"Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\nGenerate a search query based on the conversation and the new question.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return the original user question.\nDO NOT return anything besides the query."},{"role":"user","content":"good options for climbing gear that can be used outside?"},{"role":"assistant","tool_calls":[{"id":"call_abc123","type":"function","function":{"arguments":"{\"search_query\":\"climbing gear outside\"}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc123","content":"Search results for climbing gear that can be used outside: ..."},{"role":"user","content":"are there any shoes less than $50?"},{"role":"assistant","tool_calls":[{"id":"call_abc456","type":"function","function":{"arguments":"{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc456","content":"Search results for shoes cheaper than 50: ..."},{"role":"user","content":"What is the capital of France?"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":[{"role":"system","content":"Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51]."},{"role":"user","content":"What is the capital of France?\n\nSources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\n\n"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}}],"followup_questions":null},"sessionState":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":[{"role":"system","content":"Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\nGenerate a search query based on the conversation and the new question.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return the original user question.\nDO NOT return anything besides the query.\n"},{"role":"user","content":"good options for climbing gear that can be used outside?"},{"role":"assistant","tool_calls":[{"id":"call_abc123","type":"function","function":{"arguments":"{\"search_query\":\"climbing gear outside\"}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc123","content":"Search results for climbing gear that can be used outside: ..."},{"role":"user","content":"are there any shoes less than $50?"},{"role":"assistant","tool_calls":[{"id":"call_abc456","type":"function","function":{"arguments":"{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc456","content":"Search results for shoes cheaper than 50: ..."},{"role":"user","content":"What is the capital of France?"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":[{"role":"system","content":"Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51]."},{"role":"user","content":"What is the capital of France?\n\nSources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\n\n"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}}],"followup_questions":null},"sessionState":null} {"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index cd221fbb..15d6b5d8 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -45,7 +45,7 @@ async def test_similar_handler(test_client): "id": 71, "name": "Explorer Frost Boots", "price": 149.99, - "distance": 0.47, + "distance": 0.11, "type": "Footwear", "brand": "Daybird", "description": "The Explorer Frost Boots by Daybird are the perfect companion for " diff --git a/tests/test_postgres_engine.py b/tests/test_postgres_engine.py index 2bef4d6a..05f1507d 100644 --- a/tests/test_postgres_engine.py +++ b/tests/test_postgres_engine.py @@ -21,9 +21,9 @@ async def test_create_postgres_engine(mock_session_env, mock_azure_credential): azure_credential=mock_azure_credential, ) assert engine.url.host == "localhost" - assert engine.url.username == "admin" - assert engine.url.database == "postgres" - assert engine.url.password == "postgres" + assert engine.url.username == os.environ["POSTGRES_USERNAME"] + assert engine.url.database == os.environ["POSTGRES_DATABASE"] + assert engine.url.password == os.environ.get("POSTGRES_PASSWORD") assert engine.url.query["ssl"] == "prefer" @@ -33,9 +33,9 @@ async def test_create_postgres_engine_from_env(mock_session_env, mock_azure_cred azure_credential=mock_azure_credential, ) assert engine.url.host == "localhost" - assert engine.url.username == "admin" - assert engine.url.database == "postgres" - assert engine.url.password == "postgres" + assert engine.url.username == os.environ["POSTGRES_USERNAME"] + assert engine.url.database == os.environ["POSTGRES_DATABASE"] + assert engine.url.password == os.environ.get("POSTGRES_PASSWORD") assert engine.url.query["ssl"] == "prefer" @@ -57,7 +57,7 @@ async def test_create_postgres_engine_from_args(mock_azure_credential): azure_credential=mock_azure_credential, ) assert engine.url.host == "localhost" - assert engine.url.username == "admin" - assert engine.url.database == "postgres" - assert engine.url.password == "postgres" + assert engine.url.username == os.environ["POSTGRES_USERNAME"] + assert engine.url.database == os.environ["POSTGRES_DATABASE"] + assert engine.url.password == os.environ.get("POSTGRES_PASSWORD") assert engine.url.query["ssl"] == "prefer"