From e458605b93c009934863558a0be702996e63d184 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 18:26:07 +0000 Subject: [PATCH 01/55] Test evaluation --- src/backend/fastapi_app/prompts/query.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/fastapi_app/prompts/query.txt b/src/backend/fastapi_app/prompts/query.txt index 672b03be..d8cdbfc6 100644 --- a/src/backend/fastapi_app/prompts/query.txt +++ b/src/backend/fastapi_app/prompts/query.txt @@ -3,4 +3,4 @@ You have access to an Azure PostgreSQL database with an items table that has col Generate a search query based on the conversation and the new question. If the question is not in English, translate the question to English before generating the search query. If you cannot generate a search query, return the original user question. -DO NOT return anything besides the query. \ No newline at end of file +DO NOT return anything besides the query! From e670c952b3eb364e733c9bfe467751717c00f438 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 18:29:06 +0000 Subject: [PATCH 02/55] Dont run Python tests right now --- .github/workflows/app-tests.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index ff5e930f..7e2747ba 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -1,20 +1,20 @@ name: App Tests on: - push: - branches: [ main ] - paths-ignore: - - "**.md" - - ".azdo/**" - - ".devcontainer/**" - - ".github/**" - pull_request: - branches: [ main ] - paths-ignore: - - "**.md" - - ".azdo/**" - - ".devcontainer/**" - - ".github/**" + # push: + # branches: [ main ] + # paths-ignore: + # - "**.md" + # - ".azdo/**" + # - ".devcontainer/**" + # - ".github/**" + # pull_request: + # branches: [ main ] + # paths-ignore: + # - "**.md" + # - ".azdo/**" + # - ".devcontainer/**" + # - ".github/**" workflow_dispatch: permissions: From c66a8fcf8ce1c04ca2093b583af4a4eb7d3d2cb6 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 18:36:55 +0000 Subject: [PATCH 03/55] get eval working --- .github/workflows/evaluate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 73220be3..c0d4333a 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -13,7 +13,7 @@ permissions: jobs: evaluate: - if: github.event_name == 'workflow_dispatch' || contains(github.event.comment.body, '#evaluate') + #if: github.event_name == 'workflow_dispatch' || contains(github.event.comment.body, '#evaluate') runs-on: ubuntu-latest env: AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} From 16dfa1337d70b36457a194b96750c29ecfdfe4dd Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 18:38:59 +0000 Subject: [PATCH 04/55] Try to get it working --- .github/workflows/evaluate.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index c0d4333a..02872327 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -4,6 +4,8 @@ on: workflow_dispatch: issue_comment: types: [created] + pull_request: + branches: [ main ] # Set up permissions for deploying with secretless Azure federated credentials # https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication From c7258afd99cf27fcf352549a3f359509a4d2d7ed Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 18:44:35 +0000 Subject: [PATCH 05/55] Try to get it working --- .github/workflows/evaluate.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 02872327..189656f0 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -109,11 +109,6 @@ jobs: --tenant-id "$Env:AZURE_TENANT_ID" shell: pwsh - - name: Provision Infrastructure - run: azd provision --no-prompt - env: - AZD_INITIAL_ENVIRONMENT_CONFIG: ${{ secrets.AZD_INITIAL_ENVIRONMENT_CONFIG }} - - name: Run local server in background run: | RUNNER_TRACKING_ID="" && (nohup python3 -m uvicorn fastapi_app:create_app --factory > serverlogs.out 2> serverlogs.err &) From e19ec6113c7e7764eca29ab90c97874e54ee6678 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 18:58:17 +0000 Subject: [PATCH 06/55] Fix version output --- infra/main.bicep | 2 +- src/backend/fastapi_app/openai_clients.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/main.bicep b/infra/main.bicep index d8b19355..585d1910 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -447,7 +447,7 @@ output OPENAI_EMBED_HOST string = openAIEmbedHost output AZURE_OPENAI_ENDPOINT string = !empty(azureOpenAIEndpoint) ? azureOpenAIEndpoint : (deployAzureOpenAI ? openAI.outputs.endpoint : '') -output AZURE_OPENAI_VERSION string = openAIEmbedHost == 'chat' ? azureOpenAIAPIVersion : '' +output AZURE_OPENAI_VERSION string = azureOpenAIAPIVersion output AZURE_OPENAI_CHAT_DEPLOYMENT string = deployAzureOpenAI ? chatDeploymentName : '' output AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION string = deployAzureOpenAI ? chatDeploymentVersion : '' output AZURE_OPENAI_CHAT_DEPLOYMENT_CAPACITY int = deployAzureOpenAI ? chatDeploymentCapacity : 0 diff --git a/src/backend/fastapi_app/openai_clients.py b/src/backend/fastapi_app/openai_clients.py index 0610aece..10d2f506 100644 --- a/src/backend/fastapi_app/openai_clients.py +++ b/src/backend/fastapi_app/openai_clients.py @@ -62,7 +62,7 @@ async def create_openai_embed_client( openai_embed_client: openai.AsyncAzureOpenAI | openai.AsyncOpenAI OPENAI_EMBED_HOST = os.getenv("OPENAI_EMBED_HOST") if OPENAI_EMBED_HOST == "azure": - api_version = os.environ["AZURE_OPENAI_VERSION"] + api_version = os.environ["AZURE_OPENAI_VERSION"] or "2024-03-01-preview" azure_endpoint = os.environ["AZURE_OPENAI_ENDPOINT"] azure_deployment = os.environ["AZURE_OPENAI_EMBED_DEPLOYMENT"] if api_key := os.getenv("AZURE_OPENAI_KEY"): From 1d5cd6e30316919b75f58fe73fee131144162864 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 20:43:44 +0000 Subject: [PATCH 07/55] Update pgvector --- src/backend/fastapi_app/postgres_engine.py | 6 ++++++ src/backend/fastapi_app/postgres_searcher.py | 4 ++-- src/backend/pyproject.toml | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/backend/fastapi_app/postgres_engine.py b/src/backend/fastapi_app/postgres_engine.py index 364dc3e9..ecf1e2ef 100644 --- a/src/backend/fastapi_app/postgres_engine.py +++ b/src/backend/fastapi_app/postgres_engine.py @@ -2,7 +2,9 @@ import os from azure.identity import AzureDeveloperCliCredential +from pgvector.asyncpg import register_vector from sqlalchemy import event +from sqlalchemy.engine import AdaptedConnection from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine from fastapi_app.dependencies import get_azure_credential @@ -35,6 +37,10 @@ def get_password_from_azure_credential(): echo=False, ) + @event.listens_for(engine.sync_engine, "connect") + def register_custom_types(dbapi_connection: AdaptedConnection, *args): + dbapi_connection.run_async(register_vector) + @event.listens_for(engine.sync_engine, "do_connect") def update_password_token(dialect, conn_rec, cargs, cparams): if token_based_password: diff --git a/src/backend/fastapi_app/postgres_searcher.py b/src/backend/fastapi_app/postgres_searcher.py index 1953d2a1..82dd5b63 100644 --- a/src/backend/fastapi_app/postgres_searcher.py +++ b/src/backend/fastapi_app/postgres_searcher.py @@ -1,5 +1,5 @@ +import numpy as np from openai import AsyncAzureOpenAI, AsyncOpenAI -from pgvector.utils import to_db from sqlalchemy import Float, Integer, column, select, text from sqlalchemy.ext.asyncio import AsyncSession @@ -87,7 +87,7 @@ async def search( results = ( await self.db_session.execute( sql, - {"embedding": to_db(query_vector), "query": query_text, "k": 60}, + {"embedding": np.array(query_vector), "query": query_text, "k": 60}, ) ).fetchall() diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index d697f4d7..acaca112 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "aiohttp>=3.9.5,<4.0.0", "asyncpg>=0.29.0,<1.0.0", "SQLAlchemy[asyncio]>=2.0.30,<3.0.0", - "pgvector>=0.2.5,<0.3.0", + "pgvector>=0.3.0,<0.4.0", "openai>=1.34.0,<2.0.0", "tiktoken>=0.7.0,<0.8.0", "openai-messages-token-helper>=0.1.8,<0.2.0", From 61b9098ac13d0587d92a1416436f9b873f7c061c Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 20:56:43 +0000 Subject: [PATCH 08/55] Fix pgvector installation --- .github/workflows/evaluate.yaml | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 189656f0..1fea22d5 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -30,19 +30,17 @@ jobs: echo "Comment contains #evaluate hashtag" - uses: actions/checkout@v4 - - name: Install PostgreSQL development libraries + + - name: Install pgvector run: | - sudo apt update - sudo apt install postgresql-server-dev-14 - - name: Setup postgres - uses: ikalnytskyi/action-setup-postgres@v6 - with: - username: admin - password: postgres - database: postgres + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get install postgresql-14-pgvector + + - name: Start postgres + run: sudo systemctl start postgresql - - name: Install pgvector on MacOS/Linux using install-pgvector.sh - run: .github/workflows/install-pgvector.sh + - name: Create vector extension + - run: sudo -u postgres psql -c 'CREATE EXTENSION vector' - name: Install python uses: actions/setup-python@v5 From be784b54e0792048bf02a3f05f5266f77fd0b5d4 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 20:58:40 +0000 Subject: [PATCH 09/55] Fix pgvector installation --- .github/workflows/evaluate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 1fea22d5..1c6c2636 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -40,7 +40,7 @@ jobs: run: sudo systemctl start postgresql - name: Create vector extension - - run: sudo -u postgres psql -c 'CREATE EXTENSION vector' + run: sudo -u postgres psql -c 'CREATE EXTENSION vector' - name: Install python uses: actions/setup-python@v5 From 7bb35991a3369d02436b8d488b19069a602997af Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 21:02:09 +0000 Subject: [PATCH 10/55] Change username --- .github/workflows/evaluate.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 1c6c2636..13108d8d 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -64,7 +64,7 @@ jobs: python ./src/backend/fastapi_app/setup_postgres_seeddata.py env: POSTGRES_HOST: localhost - POSTGRES_USERNAME: admin + POSTGRES_USERNAME: postgres POSTGRES_PASSWORD: postgres POSTGRES_DATABASE: postgres POSTGRES_SSL: disable @@ -122,7 +122,7 @@ jobs: AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }} AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} POSTGRES_HOST: localhost - POSTGRES_USERNAME: admin + POSTGRES_USERNAME: postgres POSTGRES_PASSWORD: postgres POSTGRES_DATABASE: postgres POSTGRES_SSL: disable From a70599243af49fa980b187b8c3e2a65712519e51 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 21:08:38 +0000 Subject: [PATCH 11/55] Empty password --- .github/workflows/evaluate.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 13108d8d..599eab63 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -65,7 +65,7 @@ jobs: env: POSTGRES_HOST: localhost POSTGRES_USERNAME: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_PASSWORD: POSTGRES_DATABASE: postgres POSTGRES_SSL: disable @@ -123,7 +123,7 @@ jobs: AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} POSTGRES_HOST: localhost POSTGRES_USERNAME: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_PASSWORD: POSTGRES_DATABASE: postgres POSTGRES_SSL: disable - name: Evaluate local RAG flow From 30e27ef1f7f299a9d35c64eaaa8267ae6a56cdf3 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 21:12:46 +0000 Subject: [PATCH 12/55] Add a password --- .github/workflows/evaluate.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 599eab63..63f56de3 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -39,6 +39,9 @@ jobs: - name: Start postgres run: sudo systemctl start postgresql + - name: Set password for postgres user + run: sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'root'" + - name: Create vector extension run: sudo -u postgres psql -c 'CREATE EXTENSION vector' @@ -65,7 +68,7 @@ jobs: env: POSTGRES_HOST: localhost POSTGRES_USERNAME: postgres - POSTGRES_PASSWORD: + POSTGRES_PASSWORD: root POSTGRES_DATABASE: postgres POSTGRES_SSL: disable @@ -123,7 +126,7 @@ jobs: AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} POSTGRES_HOST: localhost POSTGRES_USERNAME: postgres - POSTGRES_PASSWORD: + POSTGRES_PASSWORD: root POSTGRES_DATABASE: postgres POSTGRES_SSL: disable - name: Evaluate local RAG flow From c7907a6b9f150e6e84a90306073cdb4f69e6d46d Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 21:32:39 +0000 Subject: [PATCH 13/55] Use np.array when committing --- src/backend/fastapi_app/setup_postgres_seeddata.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/fastapi_app/setup_postgres_seeddata.py b/src/backend/fastapi_app/setup_postgres_seeddata.py index 28956c58..05a37cf8 100644 --- a/src/backend/fastapi_app/setup_postgres_seeddata.py +++ b/src/backend/fastapi_app/setup_postgres_seeddata.py @@ -4,6 +4,7 @@ import logging import os +import numpy as np import sqlalchemy.exc from dotenv import load_dotenv from sqlalchemy import select, text @@ -41,8 +42,11 @@ async def seed_data(engine): if db_item.scalars().first(): continue attrs = {key: value for key, value in seed_data_object.items()} - row = Item(**attrs) - session.add(row) + attrs["embedding_ada002"] = np.array(seed_data_object["embedding_ada002"]) + attrs["embedding_nomic"] = np.array(seed_data_object["embedding_nomic"]) + column_names = ", ".join(attrs.keys()) + values = ", ".join([f":{key}" for key in attrs.keys()]) + await session.execute(text(f"INSERT INTO {table_name} ({column_names}) VALUES ({values})"), attrs) try: await session.commit() except sqlalchemy.exc.IntegrityError: From a62bc2f6fd90c2ec665b8839441ccdeb693a4bba Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 21:46:55 +0000 Subject: [PATCH 14/55] Move env --- .github/workflows/evaluate.yaml | 47 ++++++++++++--------------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 63f56de3..f9e81be7 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -23,6 +23,21 @@ jobs: AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} + POSTGRES_HOST: localhost + POSTGRES_USERNAME: postgres + POSTGRES_PASSWORD: root + POSTGRES_DATABASE: postgres + POSTGRES_SSL: disable + OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} + OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} + AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} + AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} + AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }} + AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }} + AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }} + AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} steps: - name: Check for evaluate hash tag if: contains(github.event.comment.body, '#evaluate') @@ -65,12 +80,6 @@ jobs: run: | python ./src/backend/fastapi_app/setup_postgres_database.py python ./src/backend/fastapi_app/setup_postgres_seeddata.py - env: - POSTGRES_HOST: localhost - POSTGRES_USERNAME: postgres - POSTGRES_PASSWORD: root - POSTGRES_DATABASE: postgres - POSTGRES_SSL: disable - name: Setup node uses: actions/setup-node@v4 @@ -113,33 +122,11 @@ jobs: - name: Run local server in background run: | RUNNER_TRACKING_ID="" && (nohup python3 -m uvicorn fastapi_app:create_app --factory > serverlogs.out 2> serverlogs.err &) - env: - OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} - OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }} - AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} - AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} - AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} - AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }} - AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }} - AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }} - AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} - POSTGRES_HOST: localhost - POSTGRES_USERNAME: postgres - POSTGRES_PASSWORD: root - POSTGRES_DATABASE: postgres - POSTGRES_SSL: disable + - name: Evaluate local RAG flow run: | python evals/evaluate.py - env: - OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }} - AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} - AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }} - AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }} - AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }} - AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }} - AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }} + - name: Upload server logs as build artifact uses: actions/upload-artifact@v4 with: From 1677f0f8d14d3908d2eb57f551fbb87ac23ec75c Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 21:58:54 +0000 Subject: [PATCH 15/55] Move env --- .github/workflows/evaluate.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index f9e81be7..f69ef25c 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -38,6 +38,8 @@ jobs: AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }} AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }} AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }} + AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }} + AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }} steps: - name: Check for evaluate hash tag if: contains(github.event.comment.body, '#evaluate') From 7874def685e151c6a01a8af807ada45db389cba6 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 9 Oct 2024 22:46:16 +0000 Subject: [PATCH 16/55] Add test query --- .../fastapi_app/setup_postgres_seeddata.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/backend/fastapi_app/setup_postgres_seeddata.py b/src/backend/fastapi_app/setup_postgres_seeddata.py index 05a37cf8..134c3a55 100644 --- a/src/backend/fastapi_app/setup_postgres_seeddata.py +++ b/src/backend/fastapi_app/setup_postgres_seeddata.py @@ -53,6 +53,22 @@ async def seed_data(engine): pass logger.info(f"{table_name} table seeded successfully.") + # Do a simple query with <=> + # Check cosine distance of every item with the first item + async with async_sessionmaker(engine, expire_on_commit=False)() as session: + first_item = (await session.execute(select(Item).order_by(Item.id).limit(1))).scalars().first() + result = ( + await session.execute( + text( + f"SELECT id, {Item.__tablename__}.embedding_ada002 <=> :embedding AS distance " + f"FROM {Item.__tablename__} ORDER BY distance LIMIT 2" + ), + {"embedding": first_item.embedding_ada002}, + ) + ).fetchall() + logger.info("Test query: cosine distance of first two items with the first item:") + for row in result: + logger.info(row) async def main(): From d7a32420e59b8c58d37d6cf0f003d8029f70381f Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 10 Oct 2024 21:37:40 +0000 Subject: [PATCH 17/55] Use uv --- .github/workflows/evaluate.yaml | 15 +++++++++++---- README.md | 2 +- src/backend/fastapi_app/postgres_engine.py | 1 + 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index f69ef25c..41861843 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -18,6 +18,7 @@ jobs: #if: github.event_name == 'workflow_dispatch' || contains(github.event.comment.body, '#evaluate') runs-on: ubuntu-latest env: + UV_SYSTEM_PYTHON: 1 AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} @@ -67,16 +68,23 @@ jobs: with: python-version: '3.12' + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + version: "0.4.20" + cache-dependency-glob: "requirements**.txt" + - name: Install azd uses: Azure/setup-azd@v1.0.0 - name: Install dependencies run: | - python -m pip install -r requirements-dev.txt + uv pip install -r requirements-dev.txt - name: Install app as editable app run: | - python -m pip install -e src/backend + uv pip install -e src/backend - name: Setup local database with seed data run: | @@ -96,8 +104,7 @@ jobs: - name: Install python packages run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt + uv pip install -r requirements-dev.txt - name: Login to Azure uses: azure/login@v2 diff --git a/README.md b/README.md index 43069b89..6ede71d3 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ Since the local app uses OpenAI models, you should first deploy it for the optim ``` 3. To use OpenAI.com OpenAI, set `OPENAI_CHAT_HOST` and `OPENAI_EMBED_HOST` to "openai". Then fill in the value for `OPENAICOM_KEY`. -4. To use Ollama, set `OPENAI_CHAT_HOST` to "ollama". Then update the values for `OLLAMA_ENDPOINT` and `OLLAMA_CHAT_MODEL` to match your local setup and model. Note that most Ollama models are not compatible with the "Advanced flow", due to the need for function calling support, so you'll need to disable that in _Developer Settings_ in the UI. In addition, the database rows are embedded using the default OpenAI embedding model, so you can't search them using an Ollama embedding model. You can either choose to set `OPENAI_EMBED_HOST` to "azure" or "openai", or turn off vector search in _Developer Settings_. +4. To use Ollama, set `OPENAI_CHAT_HOST` to "ollama". Then update the values for `OLLAMA_ENDPOINT` and `OLLAMA_CHAT_MODEL` to match your local setup and model. We recommend using "llama3.1" for the chat model, since it has support for function calling, and "nomic-embed-text" for the embedding model, since the sample data has already been embedded with this model. If you cannot use function calling, then turn off "Advanced flow" in the Developer Settings. If you cannot use the embedding model, then turn off vector search in the Developer Settings. ### Running the frontend and backend diff --git a/src/backend/fastapi_app/postgres_engine.py b/src/backend/fastapi_app/postgres_engine.py index ecf1e2ef..4587f5e1 100644 --- a/src/backend/fastapi_app/postgres_engine.py +++ b/src/backend/fastapi_app/postgres_engine.py @@ -39,6 +39,7 @@ def get_password_from_azure_credential(): @event.listens_for(engine.sync_engine, "connect") def register_custom_types(dbapi_connection: AdaptedConnection, *args): + logger.info("Registering pgvector extension...") dbapi_connection.run_async(register_vector) @event.listens_for(engine.sync_engine, "do_connect") From 7666b7f0303ac0a2b6843bcff8a45fc844b0a417 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 10 Oct 2024 21:48:26 +0000 Subject: [PATCH 18/55] Install less --- .github/workflows/evaluate.yaml | 8 ++++---- evals/requirements.txt | 1 + requirements-dev.txt | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 evals/requirements.txt diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 41861843..2ebc8d31 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -102,10 +102,6 @@ jobs: npm install npm run build - - name: Install python packages - run: | - uv pip install -r requirements-dev.txt - - name: Login to Azure uses: azure/login@v2 with: @@ -132,6 +128,10 @@ jobs: run: | RUNNER_TRACKING_ID="" && (nohup python3 -m uvicorn fastapi_app:create_app --factory > serverlogs.out 2> serverlogs.err &) + - name: Install evaluate dependencies + run: | + uv pip install -r evals/requirements.txt + - name: Evaluate local RAG flow run: | python evals/evaluate.py diff --git a/evals/requirements.txt b/evals/requirements.txt new file mode 100644 index 00000000..7559267e --- /dev/null +++ b/evals/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/Azure-Samples/ai-rag-chat-evaluator/@installable diff --git a/requirements-dev.txt b/requirements-dev.txt index 1169d8a2..c74069ff 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ -r src/backend/requirements.txt +-r evals/requirements.txt ruff mypy types-requests @@ -12,5 +13,3 @@ pytest-cov pytest-playwright pytest-snapshot locust -git+https://github.com/Azure-Samples/ai-rag-chat-evaluator/@installable -psycopg2 From f34e6c2e64e5705eb13e45e5390f525ac4b5b31d Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 10 Oct 2024 21:55:14 +0000 Subject: [PATCH 19/55] Move test query to the flow --- src/backend/fastapi_app/postgres_engine.py | 2 +- src/backend/fastapi_app/postgres_searcher.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/backend/fastapi_app/postgres_engine.py b/src/backend/fastapi_app/postgres_engine.py index 4587f5e1..763d9980 100644 --- a/src/backend/fastapi_app/postgres_engine.py +++ b/src/backend/fastapi_app/postgres_engine.py @@ -34,7 +34,7 @@ def get_password_from_azure_credential(): engine = create_async_engine( DATABASE_URI, - echo=False, + echo=True, ) @event.listens_for(engine.sync_engine, "connect") diff --git a/src/backend/fastapi_app/postgres_searcher.py b/src/backend/fastapi_app/postgres_searcher.py index 82dd5b63..b12a1b34 100644 --- a/src/backend/fastapi_app/postgres_searcher.py +++ b/src/backend/fastapi_app/postgres_searcher.py @@ -84,6 +84,18 @@ async def search( else: raise ValueError("Both query text and query vector are empty") + first_item = (await self.db_session.execute(select(Item).order_by(Item.id).limit(1))).scalars().first() + # Will it work? + ( + await self.db_session.execute( + text( + f"SELECT id, {Item.__tablename__}.embedding_ada002 <=> :embedding AS distance " + f"FROM {Item.__tablename__} ORDER BY distance LIMIT 2" + ), + {"embedding": first_item.embedding_ada002}, + ) + ).fetchall() + results = ( await self.db_session.execute( sql, From 193f5f261ae2b5489bbeb102f4ce3db2cf642066 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 10 Oct 2024 23:10:14 +0000 Subject: [PATCH 20/55] Default values --- src/backend/fastapi_app/dependencies.py | 27 +++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/backend/fastapi_app/dependencies.py b/src/backend/fastapi_app/dependencies.py index d3d0177c..5876c1eb 100644 --- a/src/backend/fastapi_app/dependencies.py +++ b/src/backend/fastapi_app/dependencies.py @@ -37,34 +37,35 @@ class FastAPIAppContext(BaseModel): async def common_parameters(): """ Get the common parameters for the FastAPI app + Use the pattern of `os.getenv("VAR_NAME") or "default_value"` to avoid empty string values """ OPENAI_EMBED_HOST = os.getenv("OPENAI_EMBED_HOST") OPENAI_CHAT_HOST = os.getenv("OPENAI_CHAT_HOST") if OPENAI_EMBED_HOST == "azure": - openai_embed_deployment = os.getenv("AZURE_OPENAI_EMBED_DEPLOYMENT", "text-embedding-ada-002") - openai_embed_model = os.getenv("AZURE_OPENAI_EMBED_MODEL", "text-embedding-ada-002") - openai_embed_dimensions = int(os.getenv("AZURE_OPENAI_EMBED_DIMENSIONS", 1536)) - embedding_column = os.getenv("AZURE_OPENAI_EMBEDDING_COLUMN", "embedding_ada002") + openai_embed_deployment = os.getenv("AZURE_OPENAI_EMBED_DEPLOYMENT") or "text-embedding-ada-002" + openai_embed_model = os.getenv("AZURE_OPENAI_EMBED_MODEL") or "text-embedding-ada-002" + openai_embed_dimensions = int(os.getenv("AZURE_OPENAI_EMBED_DIMENSIONS") or 1536) + embedding_column = os.getenv("AZURE_OPENAI_EMBEDDING_COLUMN") or "embedding_ada002" elif OPENAI_EMBED_HOST == "ollama": openai_embed_deployment = None - openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL", "nomic-embed-text") + openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL") or "nomic-embed-text" openai_embed_dimensions = None - embedding_column = os.getenv("OLLAMA_EMBEDDING_COLUMN", "embedding_nomic") + embedding_column = os.getenv("OLLAMA_EMBEDDING_COLUMN") or "embedding_nomic" else: openai_embed_deployment = None - openai_embed_model = os.getenv("OPENAICOM_EMBED_MODEL", "text-embedding-ada-002") + openai_embed_model = os.getenv("OPENAICOM_EMBED_MODEL") or "text-embedding-ada-002" openai_embed_dimensions = int(os.getenv("OPENAICOM_EMBED_DIMENSIONS", 1536)) - embedding_column = os.getenv("OPENAICOM_EMBEDDING_COLUMN", "embedding_ada002") + embedding_column = os.getenv("OPENAICOM_EMBEDDING_COLUMN") or "embedding_ada002" if OPENAI_CHAT_HOST == "azure": - openai_chat_deployment = os.getenv("AZURE_OPENAI_CHAT_DEPLOYMENT", "gpt-4o-mini") - openai_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL", "gpt-4o-mini") + openai_chat_deployment = os.getenv("AZURE_OPENAI_CHAT_DEPLOYMENT") or "gpt-4o-mini" + openai_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL") or "gpt-4o-mini" elif OPENAI_CHAT_HOST == "ollama": openai_chat_deployment = None - openai_chat_model = os.getenv("OLLAMA_CHAT_MODEL", "phi3:3.8b") - openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL", "nomic-embed-text") + openai_chat_model = os.getenv("OLLAMA_CHAT_MODEL") or "phi3:3.8b" + openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL") or "nomic-embed-text" else: openai_chat_deployment = None - openai_chat_model = os.getenv("OPENAICOM_CHAT_MODEL", "gpt-3.5-turbo") + openai_chat_model = os.getenv("OPENAICOM_CHAT_MODEL") or "gpt-3.5-turbo" return FastAPIAppContext( openai_chat_model=openai_chat_model, openai_embed_model=openai_embed_model, From 7fd2337751c3d8303b9a2c5990ada314bd9b8470 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 18 Oct 2024 22:57:21 +0000 Subject: [PATCH 21/55] Add markdown output --- .github/workflows/evaluate.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 2ebc8d31..382ef39b 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -156,9 +156,6 @@ jobs: - name: GitHub Summary Step if: ${{ success() }} - working-directory: ./src/api run: | - echo "" >> $GITHUB_STEP_SUMMARY - - echo "📊 Promptflow Evaluation Results" >> $GITHUB_STEP_SUMMARY - cat evaluate/eval_results.md >> $GITHUB_STEP_SUMMARY + echo "📊 Evaluation Results" >> $GITHUB_STEP_SUMMARY + python -m evaltools summary evals/results --output=markdown >> $GITHUB_STEP_SUMMARY From 8205a0a344c83aa863482f094b0e585429d4d257 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 18 Oct 2024 23:03:05 +0000 Subject: [PATCH 22/55] Add markdown output --- .github/workflows/evaluate.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 382ef39b..1f03c56a 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -6,6 +6,8 @@ on: types: [created] pull_request: branches: [ main ] + push: + branches: [ main, test-eval ] # Set up permissions for deploying with secretless Azure federated credentials # https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication From 3f4e5f841fc86f601e408d523586bdface313620 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 18 Oct 2024 23:09:19 +0000 Subject: [PATCH 23/55] Configure Azure Developer Pipeline --- .github/workflows/evaluate.yaml | 44 +++++++++++----------- src/backend/fastapi_app/postgres_engine.py | 5 +-- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 1f03c56a..20bdc3b9 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -80,6 +80,28 @@ jobs: - name: Install azd uses: Azure/setup-azd@v1.0.0 + - name: Login to Azure + uses: azure/login@v2 + with: + client-id: ${{ env.AZURE_CLIENT_ID }} + tenant-id: ${{ env.AZURE_TENANT_ID }} + subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }} + + - name: Set az account + uses: azure/CLI@v2 + with: + inlineScript: | + az account set --subscription ${{env.AZURE_SUBSCRIPTION_ID}} + + - name: Log in with Azure (Federated Credentials) + if: ${{ env.AZURE_CLIENT_ID != '' }} + run: | + azd auth login ` + --client-id "$Env:AZURE_CLIENT_ID" ` + --federated-credential-provider "github" ` + --tenant-id "$Env:AZURE_TENANT_ID" + shell: pwsh + - name: Install dependencies run: | uv pip install -r requirements-dev.txt @@ -104,28 +126,6 @@ jobs: npm install npm run build - - name: Login to Azure - uses: azure/login@v2 - with: - client-id: ${{ env.AZURE_CLIENT_ID }} - tenant-id: ${{ env.AZURE_TENANT_ID }} - subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }} - - - name: Set az account - uses: azure/CLI@v2 - with: - inlineScript: | - az account set --subscription ${{env.AZURE_SUBSCRIPTION_ID}} - - - name: Log in with Azure (Federated Credentials) - if: ${{ env.AZURE_CLIENT_ID != '' }} - run: | - azd auth login ` - --client-id "$Env:AZURE_CLIENT_ID" ` - --federated-credential-provider "github" ` - --tenant-id "$Env:AZURE_TENANT_ID" - shell: pwsh - - name: Run local server in background run: | RUNNER_TRACKING_ID="" && (nohup python3 -m uvicorn fastapi_app:create_app --factory > serverlogs.out 2> serverlogs.err &) diff --git a/src/backend/fastapi_app/postgres_engine.py b/src/backend/fastapi_app/postgres_engine.py index 763d9980..1b91a10e 100644 --- a/src/backend/fastapi_app/postgres_engine.py +++ b/src/backend/fastapi_app/postgres_engine.py @@ -32,10 +32,7 @@ def get_password_from_azure_credential(): if sslmode: DATABASE_URI += f"?ssl={sslmode}" - engine = create_async_engine( - DATABASE_URI, - echo=True, - ) + engine = create_async_engine(DATABASE_URI, echo=False) @event.listens_for(engine.sync_engine, "connect") def register_custom_types(dbapi_connection: AdaptedConnection, *args): From 8e4453d1e6d6f4ac5d05466de9e3286827fd3944 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 18 Oct 2024 23:22:47 +0000 Subject: [PATCH 24/55] Evaluate 10 questions --- evals/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/evaluate.py b/evals/evaluate.py index 812a3f09..2a18b753 100644 --- a/evals/evaluate.py +++ b/evals/evaluate.py @@ -56,5 +56,5 @@ def get_openai_config() -> dict: # TODO: Specify the num questions using argument run_evaluate_from_config( - working_dir=Path(__file__).parent, config_path="eval_config.json", openai_config=openai_config, num_questions=2 + working_dir=Path(__file__).parent, config_path="eval_config.json", openai_config=openai_config, num_questions=10 ) From 5148cdf7eec599125111e2f4cdf351fe40b4f27a Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 18 Oct 2024 23:28:31 +0000 Subject: [PATCH 25/55] Run on all questions --- evals/evaluate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/evaluate.py b/evals/evaluate.py index 2a18b753..a0df549c 100644 --- a/evals/evaluate.py +++ b/evals/evaluate.py @@ -53,8 +53,8 @@ def get_openai_config() -> dict: openai_config = get_openai_config() # TODO: specify the localhost URL using argument # TODO: specify the experiment name (based on PR number) - # TODO: Specify the num questions using argument + # TODO: Specify the num questions using arg CLI run_evaluate_from_config( - working_dir=Path(__file__).parent, config_path="eval_config.json", openai_config=openai_config, num_questions=10 + working_dir=Path(__file__).parent, config_path="eval_config.json", openai_config=openai_config ) From af62dff20011d2d7be6f5fd30702d7b93ba20ae6 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Fri, 18 Oct 2024 23:32:01 +0000 Subject: [PATCH 26/55] Run on all questions --- evals/evaluate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/evals/evaluate.py b/evals/evaluate.py index a0df549c..41e7f01f 100644 --- a/evals/evaluate.py +++ b/evals/evaluate.py @@ -56,5 +56,8 @@ def get_openai_config() -> dict: # TODO: Specify the num questions using arg CLI run_evaluate_from_config( - working_dir=Path(__file__).parent, config_path="eval_config.json", openai_config=openai_config + working_dir=Path(__file__).parent, + config_path="eval_config.json", + openai_config=openai_config, + num_questions=None, ) From f1a05e9b6465682597ba90acfb069e3eb7b6bca1 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sun, 20 Oct 2024 10:14:22 +0000 Subject: [PATCH 27/55] Add args and PR # --- .github/workflows/evaluate.yaml | 5 +++-- evals/evaluate.py | 15 +++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 20bdc3b9..c71c926c 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -136,7 +136,8 @@ jobs: - name: Evaluate local RAG flow run: | - python evals/evaluate.py + PR_NUMBER=${{ github.event.pull_request.number }} + python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=results/pr$PR_NUMBER - name: Upload server logs as build artifact uses: actions/upload-artifact@v4 @@ -154,7 +155,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: eval_result - path: ./src/api/evaluate/eval_results.jsonl + path: ./evals/results/pr${{ github.event.pull_request.number }} - name: GitHub Summary Step if: ${{ success() }} diff --git a/evals/evaluate.py b/evals/evaluate.py index 41e7f01f..34c6e1e7 100644 --- a/evals/evaluate.py +++ b/evals/evaluate.py @@ -1,3 +1,4 @@ +import argparse import logging import os from pathlib import Path @@ -50,14 +51,20 @@ def get_openai_config() -> dict: ) load_dotenv(".env", override=True) + parser = argparse.ArgumentParser(description="Run evaluation with OpenAI configuration.") + parser.add_argument("--targeturl", type=str, help="Specify the target URL.") + parser.add_argument("--resultsdir", type=str, help="Specify the results directory.") + parser.add_argument("--numquestions", type=int, help="Specify the number of questions.") + + args = parser.parse_args() + openai_config = get_openai_config() - # TODO: specify the localhost URL using argument - # TODO: specify the experiment name (based on PR number) - # TODO: Specify the num questions using arg CLI run_evaluate_from_config( working_dir=Path(__file__).parent, config_path="eval_config.json", + num_questions=args.num_questions, + target_url=args.target_url, + results_dir=args.results_dir, openai_config=openai_config, - num_questions=None, ) From c4f7d2f28b6c73d6a94e75ed337351b23cc81a05 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sun, 20 Oct 2024 10:18:52 +0000 Subject: [PATCH 28/55] Add args and PR # --- evals/evaluate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/evals/evaluate.py b/evals/evaluate.py index 34c6e1e7..96d52615 100644 --- a/evals/evaluate.py +++ b/evals/evaluate.py @@ -63,8 +63,8 @@ def get_openai_config() -> dict: run_evaluate_from_config( working_dir=Path(__file__).parent, config_path="eval_config.json", - num_questions=args.num_questions, - target_url=args.target_url, - results_dir=args.results_dir, + num_questions=args.numquestions, + target_url=args.targeturl, + results_dir=args.resultsdir, openai_config=openai_config, ) From 80f6c52d8cb246608f76134654e5a3bd2354d9a3 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sun, 20 Oct 2024 10:24:28 +0000 Subject: [PATCH 29/55] Add args and PR # --- evals/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/evaluate.py b/evals/evaluate.py index 96d52615..9ec2eb31 100644 --- a/evals/evaluate.py +++ b/evals/evaluate.py @@ -53,7 +53,7 @@ def get_openai_config() -> dict: parser = argparse.ArgumentParser(description="Run evaluation with OpenAI configuration.") parser.add_argument("--targeturl", type=str, help="Specify the target URL.") - parser.add_argument("--resultsdir", type=str, help="Specify the results directory.") + parser.add_argument("--resultsdir", type=Path, help="Specify the results directory.") parser.add_argument("--numquestions", type=int, help="Specify the number of questions.") args = parser.parse_args() From fc103229421335a34999305082c3b94e28575a75 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sun, 20 Oct 2024 10:35:08 +0000 Subject: [PATCH 30/55] PR # --- .github/workflows/evaluate.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index c71c926c..0b945006 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -136,7 +136,7 @@ jobs: - name: Evaluate local RAG flow run: | - PR_NUMBER=${{ github.event.pull_request.number }} + PR_NUMBER=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; /^refs\/pull/ { print $3 }') python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=results/pr$PR_NUMBER - name: Upload server logs as build artifact @@ -155,7 +155,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: eval_result - path: ./evals/results/pr${{ github.event.pull_request.number }} + path: ./evals/results/pr$PR_NUMBER - name: GitHub Summary Step if: ${{ success() }} From 209d3de516a9301e12ed93e835f47a45792f596e Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sun, 20 Oct 2024 10:50:34 +0000 Subject: [PATCH 31/55] revert unneeded changes --- .github/workflows/evaluate.yaml | 12 +++--------- src/backend/fastapi_app/postgres_searcher.py | 12 ------------ src/backend/fastapi_app/prompts/query.txt | 2 +- .../fastapi_app/setup_postgres_seeddata.py | 16 ---------------- 4 files changed, 4 insertions(+), 38 deletions(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index 0b945006..1452cd11 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -1,13 +1,8 @@ name: Evaluate on: - workflow_dispatch: issue_comment: types: [created] - pull_request: - branches: [ main ] - push: - branches: [ main, test-eval ] # Set up permissions for deploying with secretless Azure federated credentials # https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication @@ -17,7 +12,7 @@ permissions: jobs: evaluate: - #if: github.event_name == 'workflow_dispatch' || contains(github.event.comment.body, '#evaluate') + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '#evaluate') runs-on: ubuntu-latest env: UV_SYSTEM_PYTHON: 1 @@ -136,8 +131,7 @@ jobs: - name: Evaluate local RAG flow run: | - PR_NUMBER=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; /^refs\/pull/ { print $3 }') - python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=results/pr$PR_NUMBER + python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=results/pr${{ github.event.issue.number }} - name: Upload server logs as build artifact uses: actions/upload-artifact@v4 @@ -155,7 +149,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: eval_result - path: ./evals/results/pr$PR_NUMBER + path: ./evals/results/pr${{ github.event.issue.number }} - name: GitHub Summary Step if: ${{ success() }} diff --git a/src/backend/fastapi_app/postgres_searcher.py b/src/backend/fastapi_app/postgres_searcher.py index b12a1b34..82dd5b63 100644 --- a/src/backend/fastapi_app/postgres_searcher.py +++ b/src/backend/fastapi_app/postgres_searcher.py @@ -84,18 +84,6 @@ async def search( else: raise ValueError("Both query text and query vector are empty") - first_item = (await self.db_session.execute(select(Item).order_by(Item.id).limit(1))).scalars().first() - # Will it work? - ( - await self.db_session.execute( - text( - f"SELECT id, {Item.__tablename__}.embedding_ada002 <=> :embedding AS distance " - f"FROM {Item.__tablename__} ORDER BY distance LIMIT 2" - ), - {"embedding": first_item.embedding_ada002}, - ) - ).fetchall() - results = ( await self.db_session.execute( sql, diff --git a/src/backend/fastapi_app/prompts/query.txt b/src/backend/fastapi_app/prompts/query.txt index d8cdbfc6..6bbb0a23 100644 --- a/src/backend/fastapi_app/prompts/query.txt +++ b/src/backend/fastapi_app/prompts/query.txt @@ -3,4 +3,4 @@ You have access to an Azure PostgreSQL database with an items table that has col Generate a search query based on the conversation and the new question. If the question is not in English, translate the question to English before generating the search query. If you cannot generate a search query, return the original user question. -DO NOT return anything besides the query! +DO NOT return anything besides the query. diff --git a/src/backend/fastapi_app/setup_postgres_seeddata.py b/src/backend/fastapi_app/setup_postgres_seeddata.py index 134c3a55..05a37cf8 100644 --- a/src/backend/fastapi_app/setup_postgres_seeddata.py +++ b/src/backend/fastapi_app/setup_postgres_seeddata.py @@ -53,22 +53,6 @@ async def seed_data(engine): pass logger.info(f"{table_name} table seeded successfully.") - # Do a simple query with <=> - # Check cosine distance of every item with the first item - async with async_sessionmaker(engine, expire_on_commit=False)() as session: - first_item = (await session.execute(select(Item).order_by(Item.id).limit(1))).scalars().first() - result = ( - await session.execute( - text( - f"SELECT id, {Item.__tablename__}.embedding_ada002 <=> :embedding AS distance " - f"FROM {Item.__tablename__} ORDER BY distance LIMIT 2" - ), - {"embedding": first_item.embedding_ada002}, - ) - ).fetchall() - logger.info("Test query: cosine distance of first two items with the first item:") - for row in result: - logger.info(row) async def main(): From 62a23fb1e2781f87589ac8949d4798735e1f1921 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sun, 20 Oct 2024 11:03:50 +0000 Subject: [PATCH 32/55] Revert workflow change --- .github/workflows/app-tests.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index fdc9e57f..c96338a2 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -1,20 +1,20 @@ name: App Tests on: - # push: - # branches: [ main ] - # paths-ignore: - # - "**.md" - # - ".azdo/**" - # - ".devcontainer/**" - # - ".github/**" - # pull_request: - # branches: [ main ] - # paths-ignore: - # - "**.md" - # - ".azdo/**" - # - ".devcontainer/**" - # - ".github/**" + push: + branches: [ main ] + paths-ignore: + - "**.md" + - ".azdo/**" + - ".devcontainer/**" + - ".github/**" + pull_request: + branches: [ main ] + paths-ignore: + - "**.md" + - ".azdo/**" + - ".devcontainer/**" + - ".github/**" workflow_dispatch: permissions: From 2c3ef8fee666ac874422304154a4ba65665fa6f0 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Sun, 20 Oct 2024 11:08:50 +0000 Subject: [PATCH 33/55] Fix pgvector --- .github/workflows/app-tests.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index c96338a2..9a57bf4e 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -52,9 +52,13 @@ jobs: username: admin password: postgres database: postgres + - name: Install pgvector on MacOS/Linux using install-pgvector.sh if: matrix.os != 'windows-latest' - run: .github/workflows/install-pgvector.sh + run: | + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get install postgresql-14-pgvector + - name: Setup python uses: actions/setup-python@v5 with: From 9832f3324c1baef15fa9e726552a4471aedc9eca Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 17:06:08 +0000 Subject: [PATCH 34/55] Get normal tests working --- .github/workflows/app-tests.yaml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 9a57bf4e..be4de689 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -27,8 +27,8 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest-xlarge", "macos-13"] - python_version: ["3.10", "3.11", "3.12"] + os: ["ubuntu-latest"] + python_version: ["3.10"] exclude: - os: macos-latest-xlarge python_version: "3.10" @@ -41,29 +41,35 @@ jobs: if: matrix.os == 'windows-latest' shell: cmd run: .github\workflows\install-pgvector.bat - - name: Install PostgreSQL development libraries - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update - sudo apt install postgresql-server-dev-14 - name: Setup postgres + if: matrix.os == 'macos-latest-xlarge' uses: ikalnytskyi/action-setup-postgres@v6 with: username: admin password: postgres database: postgres - - name: Install pgvector on MacOS/Linux using install-pgvector.sh + - name: Install pgvector if: matrix.os != 'windows-latest' run: | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y sudo apt-get install postgresql-14-pgvector + - name: Start postgres + run: sudo systemctl start postgresql + + - name: Set password for postgres user + run: sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'root'" + + - name: Create vector extension + run: sudo -u postgres psql -c 'CREATE EXTENSION vector' + - name: Setup python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} architecture: x64 + - name: Install dependencies run: | python -m pip install -r requirements-dev.txt From 699e6510f8b549d9a562ff86f1b3fe2fcdb8f651 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 17:10:33 +0000 Subject: [PATCH 35/55] Postgres env --- .github/workflows/app-tests.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index be4de689..ccb18f4b 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -24,6 +24,12 @@ jobs: test-package: name: Test ${{ matrix.os }} Python ${{ matrix.python_version }} runs-on: ${{ matrix.os }} + env: + POSTGRES_HOST: localhost + POSTGRES_USERNAME: postgres + POSTGRES_PASSWORD: root + POSTGRES_DATABASE: postgres + POSTGRES_SSL: disable strategy: fail-fast: false matrix: @@ -78,7 +84,6 @@ jobs: python -m pip install -e src/backend - name: Setup local database with seed data run: | - cp .env.sample .env python ./src/backend/fastapi_app/setup_postgres_database.py python ./src/backend/fastapi_app/setup_postgres_seeddata.py - name: Setup node From b1d7fd1b07cf8ac40b6362c9f8731deb07222ac9 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 17:17:01 +0000 Subject: [PATCH 36/55] Always use localhost --- tests/conftest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 87a1eb22..e7665237 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,10 +24,11 @@ from tests.data import test_data from tests.mocks import MockAzureCredential +# Always use localhost for testing POSTGRES_HOST = "localhost" -POSTGRES_USERNAME = "admin" -POSTGRES_DATABASE = "postgres" -POSTGRES_PASSWORD = "postgres" +POSTGRES_USERNAME = os.getenv("POSTGRES_USERNAME", "admin") +POSTGRES_DATABASE = os.getenv("POSTGRES_DATABASE", "postgres") +POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD", "postgres") POSTGRES_SSL = "prefer" POSTGRESQL_DATABASE_URL = ( f"postgresql+asyncpg://{POSTGRES_USERNAME}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}/{POSTGRES_DATABASE}" From 0fd1bc5ba9155e2d73c084367f4843f277b9b2fa Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 20:17:25 +0000 Subject: [PATCH 37/55] Update tests and similar route --- src/backend/fastapi_app/api_models.py | 4 ++++ src/backend/fastapi_app/routes/api_routes.py | 24 +++++++++++-------- .../advanced_chat_flow_response.json | 2 +- ...ced_chat_streaming_flow_response.jsonlines | 2 +- tests/test_api_routes.py | 2 +- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/backend/fastapi_app/api_models.py b/src/backend/fastapi_app/api_models.py index e1c71b93..35068a80 100644 --- a/src/backend/fastapi_app/api_models.py +++ b/src/backend/fastapi_app/api_models.py @@ -80,6 +80,10 @@ class ItemPublic(BaseModel): class ItemWithDistance(ItemPublic): distance: float + def __init__(self, **data): + super().__init__(**data) + self.distance = round(self.distance, 2) + class ChatParams(ChatRequestOverrides): prompt_template: str diff --git a/src/backend/fastapi_app/routes/api_routes.py b/src/backend/fastapi_app/routes/api_routes.py index 57cbe3a1..d1c232f0 100644 --- a/src/backend/fastapi_app/routes/api_routes.py +++ b/src/backend/fastapi_app/routes/api_routes.py @@ -5,7 +5,7 @@ import fastapi from fastapi import HTTPException from fastapi.responses import StreamingResponse -from sqlalchemy import select +from sqlalchemy import select, text from fastapi_app.api_models import ( ChatRequest, @@ -54,15 +54,19 @@ async def similar_handler( if not item: raise HTTPException(detail=f"Item with ID {id} not found.", status_code=404) - closest = await database_session.execute( - select(Item, Item.embedding_ada002.l2_distance(item.embedding_ada002)) - .filter(Item.id != id) - .order_by(Item.embedding_ada002.l2_distance(item.embedding_ada002)) - .limit(n) - ) - return [ - ItemWithDistance.model_validate(item.to_dict() | {"distance": round(distance, 2)}) for item, distance in closest - ] + closest = ( + await database_session.execute( + text( + f"SELECT *, {context.embedding_column} <=> :embedding as DISTANCE FROM {Item.__tablename__} " + "WHERE id <> :item_id ORDER BY distance LIMIT :n" + ), + {"embedding": item.embedding_ada002, "n": n, "item_id": id}, + ) + ).fetchall() + + # Convert results to ItemWithDistance models, + items = [dict(row._mapping) for row in closest] + return [ItemWithDistance.model_validate(item) for item in items] @router.get("/search", response_model=list[ItemPublic]) diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json index 7ad46375..d9f9762d 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json @@ -20,7 +20,7 @@ "description": [ { "role": "system", - "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\nGenerate a search query based on the conversation and the new question.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return the original user question.\nDO NOT return anything besides the query." + "content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\nGenerate a search query based on the conversation and the new question.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return the original user question.\nDO NOT return anything besides the query.\n" }, { "role": "user", diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines index 3615937f..9f5aaa63 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":[{"role":"system","content":"Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\nGenerate a search query based on the conversation and the new question.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return the original user question.\nDO NOT return anything besides the query."},{"role":"user","content":"good options for climbing gear that can be used outside?"},{"role":"assistant","tool_calls":[{"id":"call_abc123","type":"function","function":{"arguments":"{\"search_query\":\"climbing gear outside\"}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc123","content":"Search results for climbing gear that can be used outside: ..."},{"role":"user","content":"are there any shoes less than $50?"},{"role":"assistant","tool_calls":[{"id":"call_abc456","type":"function","function":{"arguments":"{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc456","content":"Search results for shoes cheaper than 50: ..."},{"role":"user","content":"What is the capital of France?"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":[{"role":"system","content":"Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51]."},{"role":"user","content":"What is the capital of France?\n\nSources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\n\n"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}}],"followup_questions":null},"sessionState":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":[{"role":"system","content":"Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\nGenerate a search query based on the conversation and the new question.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return the original user question.\nDO NOT return anything besides the query.\n"},{"role":"user","content":"good options for climbing gear that can be used outside?"},{"role":"assistant","tool_calls":[{"id":"call_abc123","type":"function","function":{"arguments":"{\"search_query\":\"climbing gear outside\"}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc123","content":"Search results for climbing gear that can be used outside: ..."},{"role":"user","content":"are there any shoes less than $50?"},{"role":"assistant","tool_calls":[{"id":"call_abc456","type":"function","function":{"arguments":"{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}","name":"search_database"}}]},{"role":"tool","tool_call_id":"call_abc456","content":"Search results for shoes cheaper than 50: ..."},{"role":"user","content":"What is the capital of France?"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":[{"role":"system","content":"Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51]."},{"role":"user","content":"What is the capital of France?\n\nSources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\n\n"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}}],"followup_questions":null},"sessionState":null} {"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index cd221fbb..15d6b5d8 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -45,7 +45,7 @@ async def test_similar_handler(test_client): "id": 71, "name": "Explorer Frost Boots", "price": 149.99, - "distance": 0.47, + "distance": 0.11, "type": "Footwear", "brand": "Daybird", "description": "The Explorer Frost Boots by Daybird are the perfect companion for " From 06e3bd67e9eeb9440f16d1028e52d16a4c45af70 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 20:19:02 +0000 Subject: [PATCH 38/55] Rm unneeded comment --- src/backend/fastapi_app/routes/api_routes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/fastapi_app/routes/api_routes.py b/src/backend/fastapi_app/routes/api_routes.py index d1c232f0..8dc1d8bd 100644 --- a/src/backend/fastapi_app/routes/api_routes.py +++ b/src/backend/fastapi_app/routes/api_routes.py @@ -64,7 +64,6 @@ async def similar_handler( ) ).fetchall() - # Convert results to ItemWithDistance models, items = [dict(row._mapping) for row in closest] return [ItemWithDistance.model_validate(item) for item in items] From 7db7fcd5cb1a987b9190ca5c10d29fcb3ebb55fe Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 20:22:40 +0000 Subject: [PATCH 39/55] Update PG tests --- tests/test_postgres_engine.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_postgres_engine.py b/tests/test_postgres_engine.py index 2bef4d6a..05f1507d 100644 --- a/tests/test_postgres_engine.py +++ b/tests/test_postgres_engine.py @@ -21,9 +21,9 @@ async def test_create_postgres_engine(mock_session_env, mock_azure_credential): azure_credential=mock_azure_credential, ) assert engine.url.host == "localhost" - assert engine.url.username == "admin" - assert engine.url.database == "postgres" - assert engine.url.password == "postgres" + assert engine.url.username == os.environ["POSTGRES_USERNAME"] + assert engine.url.database == os.environ["POSTGRES_DATABASE"] + assert engine.url.password == os.environ.get("POSTGRES_PASSWORD") assert engine.url.query["ssl"] == "prefer" @@ -33,9 +33,9 @@ async def test_create_postgres_engine_from_env(mock_session_env, mock_azure_cred azure_credential=mock_azure_credential, ) assert engine.url.host == "localhost" - assert engine.url.username == "admin" - assert engine.url.database == "postgres" - assert engine.url.password == "postgres" + assert engine.url.username == os.environ["POSTGRES_USERNAME"] + assert engine.url.database == os.environ["POSTGRES_DATABASE"] + assert engine.url.password == os.environ.get("POSTGRES_PASSWORD") assert engine.url.query["ssl"] == "prefer" @@ -57,7 +57,7 @@ async def test_create_postgres_engine_from_args(mock_azure_credential): azure_credential=mock_azure_credential, ) assert engine.url.host == "localhost" - assert engine.url.username == "admin" - assert engine.url.database == "postgres" - assert engine.url.password == "postgres" + assert engine.url.username == os.environ["POSTGRES_USERNAME"] + assert engine.url.database == os.environ["POSTGRES_DATABASE"] + assert engine.url.password == os.environ.get("POSTGRES_PASSWORD") assert engine.url.query["ssl"] == "prefer" From 5f09487969c2ee086b2374544a7970726637ccee Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 20:31:23 +0000 Subject: [PATCH 40/55] Bring back the matrix --- .github/workflows/app-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index ccb18f4b..7b25daff 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -33,8 +33,8 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] - python_version: ["3.10"] + os: ["ubuntu-latest", "windows-latest", "macos-latest-xlarge", "macos-13"] + python_version: ["3.10", "3.11", "3.12"] exclude: - os: macos-latest-xlarge python_version: "3.10" From 5fbb60b49e9abb41126c22674c6d0ec7e2d3f5d5 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 20:40:10 +0000 Subject: [PATCH 41/55] OSes --- .github/workflows/app-tests.yaml | 51 +++++++++++++++++++------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 7b25daff..ad4ef6c3 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -40,14 +40,17 @@ jobs: python_version: "3.10" steps: - uses: actions/checkout@v4 - - name: Check for MacOS Runner - if: matrix.os == 'macos-latest-xlarge' - run: brew install postgresql@14 - - name: Install pgvector on Windows using install-pgvector.bat - if: matrix.os == 'windows-latest' - shell: cmd - run: .github\workflows\install-pgvector.bat - - name: Setup postgres + + - name: (MacOS) Install postgreSQL and pgvector using brew + if: matrix.os == 'macos-13' + run: | + brew install postgresql@14 + brew link --overwrite postgresql@14 + brew install pgvector + brew services start postgresql@17 && sleep 1 + psql -d postgres -c 'CREATE EXTENSION vector' + + - name: (MacOSLarge) Install postgreSQL using action-setup-postgres if: matrix.os == 'macos-latest-xlarge' uses: ikalnytskyi/action-setup-postgres@v6 with: @@ -55,20 +58,19 @@ jobs: password: postgres database: postgres - - name: Install pgvector - if: matrix.os != 'windows-latest' + - name: (Windows) Install pgvector using install-pgvector.bat + if: matrix.os == 'windows-latest' + shell: cmd + run: .github\workflows\install-pgvector.bat + + - name: (Linux) Install pgvector and set password + if: matrix.os == 'ubuntu-latest' run: | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y sudo apt-get install postgresql-14-pgvector - - - name: Start postgres - run: sudo systemctl start postgresql - - - name: Set password for postgres user - run: sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'root'" - - - name: Create vector extension - run: sudo -u postgres psql -c 'CREATE EXTENSION vector' + sudo systemctl start postgresql + sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'root'" + sudo -u postgres psql -c 'CREATE EXTENSION vector' - name: Setup python uses: actions/setup-python@v5 @@ -79,36 +81,45 @@ jobs: - name: Install dependencies run: | python -m pip install -r requirements-dev.txt + - name: Install app as editable app run: | python -m pip install -e src/backend + - name: Setup local database with seed data run: | python ./src/backend/fastapi_app/setup_postgres_database.py python ./src/backend/fastapi_app/setup_postgres_seeddata.py + - name: Setup node uses: actions/setup-node@v4 with: node-version: 18 + - name: Build frontend run: | cd ./src/frontend npm install npm run build - - name: cache mypy + + - name: Setup mypy cache uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 with: path: ./.mypy_cache key: mypy${{ matrix.os }}-${{ matrix.python_version }}-${{ hashFiles('requirements-dev.txt', 'src/backend/requirements.txt', 'src/backend/pyproject.toml') }} + - name: Run MyPy run: python3 -m mypy . + - name: Run Pytest run: python3 -m pytest -s -vv --cov --cov-fail-under=85 + - name: Run E2E tests with Playwright id: e2e run: | playwright install chromium --with-deps python3 -m pytest tests/e2e.py --tracing=retain-on-failure + - name: Upload test artifacts if: ${{ failure() && steps.e2e.conclusion == 'failure' }} uses: actions/upload-artifact@v4 From 00ee507783242372a34360629960e56b1e7d74e2 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 20:42:23 +0000 Subject: [PATCH 42/55] OSes --- .github/workflows/app-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index ad4ef6c3..2d34cb60 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -47,7 +47,7 @@ jobs: brew install postgresql@14 brew link --overwrite postgresql@14 brew install pgvector - brew services start postgresql@17 && sleep 1 + brew services start postgresql@14 && sleep 1 psql -d postgres -c 'CREATE EXTENSION vector' - name: (MacOSLarge) Install postgreSQL using action-setup-postgres From 2a06e685e4c62d4b6735cf0c57b452be9a25fa67 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 20:52:28 +0000 Subject: [PATCH 43/55] More OSes --- .github/workflows/app-tests.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 2d34cb60..3d7bafc6 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -49,14 +49,15 @@ jobs: brew install pgvector brew services start postgresql@14 && sleep 1 psql -d postgres -c 'CREATE EXTENSION vector' + psql -d postgres -c "ALTER USER $(whoami) WITH PASSWORD '${{ secrets.POSTGRES_PASSWORD }}'" - name: (MacOSLarge) Install postgreSQL using action-setup-postgres if: matrix.os == 'macos-latest-xlarge' uses: ikalnytskyi/action-setup-postgres@v6 with: - username: admin - password: postgres - database: postgres + username: ${{ env.POSTGRES_USERNAME }} + password: ${{ env.POSTGRES_PASSWORD }} + database: ${{ env.POSTGRES_DATABASE }} - name: (Windows) Install pgvector using install-pgvector.bat if: matrix.os == 'windows-latest' From 62ff01c6ad6489018f8bbb6ff0ee21fad3dd2c90 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 20:58:26 +0000 Subject: [PATCH 44/55] Debug macos --- .github/workflows/app-tests.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 3d7bafc6..bdaa78d5 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -33,8 +33,10 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest-xlarge", "macos-13"] - python_version: ["3.10", "3.11", "3.12"] + #os: ["ubuntu-latest", "windows-latest", "macos-latest-xlarge", "macos-13"] + os: ["macos-13"] + python_version: ["3.10"] + #python_version: ["3.10", "3.11", "3.12"] exclude: - os: macos-latest-xlarge python_version: "3.10" @@ -48,8 +50,8 @@ jobs: brew link --overwrite postgresql@14 brew install pgvector brew services start postgresql@14 && sleep 1 - psql -d postgres -c 'CREATE EXTENSION vector' - psql -d postgres -c "ALTER USER $(whoami) WITH PASSWORD '${{ secrets.POSTGRES_PASSWORD }}'" + createuser -s ${{ env.POSTGRES_USERNAME }} + psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'" - name: (MacOSLarge) Install postgreSQL using action-setup-postgres if: matrix.os == 'macos-latest-xlarge' From 11d042df6889953fbbd2233528af617f07e09129 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:04:26 +0000 Subject: [PATCH 45/55] Mac --- .github/workflows/app-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index bdaa78d5..69969902 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -52,6 +52,7 @@ jobs: brew services start postgresql@14 && sleep 1 createuser -s ${{ env.POSTGRES_USERNAME }} psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'" + psql -d postgres -c 'CREATE EXTENSION vector' - name: (MacOSLarge) Install postgreSQL using action-setup-postgres if: matrix.os == 'macos-latest-xlarge' From 3be51abdc2cec087970ad51d2089cffcd00ed4b2 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:14:39 +0000 Subject: [PATCH 46/55] Test M1 --- .github/workflows/app-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 69969902..5714250d 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -34,8 +34,8 @@ jobs: fail-fast: false matrix: #os: ["ubuntu-latest", "windows-latest", "macos-latest-xlarge", "macos-13"] - os: ["macos-13"] - python_version: ["3.10"] + os: ["macos-latest-xlarge"] + python_version: ["3.11"] #python_version: ["3.10", "3.11", "3.12"] exclude: - os: macos-latest-xlarge From b51703c3196356b45f16336bc395a787f025d99f Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:15:49 +0000 Subject: [PATCH 47/55] Try the same for M1 --- .github/workflows/app-tests.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 5714250d..188ee377 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -44,7 +44,7 @@ jobs: - uses: actions/checkout@v4 - name: (MacOS) Install postgreSQL and pgvector using brew - if: matrix.os == 'macos-13' + if: matrix.os == 'macos-13' || matrix.os == 'macos-latest-xlarge' run: | brew install postgresql@14 brew link --overwrite postgresql@14 @@ -54,13 +54,13 @@ jobs: psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'" psql -d postgres -c 'CREATE EXTENSION vector' - - name: (MacOSLarge) Install postgreSQL using action-setup-postgres - if: matrix.os == 'macos-latest-xlarge' - uses: ikalnytskyi/action-setup-postgres@v6 - with: - username: ${{ env.POSTGRES_USERNAME }} - password: ${{ env.POSTGRES_PASSWORD }} - database: ${{ env.POSTGRES_DATABASE }} + # - name: (MacOSLarge) Install postgreSQL using action-setup-postgres + # if: matrix.os == 'macos-latest-xlarge' + # uses: ikalnytskyi/action-setup-postgres@v6 + # with: + # username: ${{ env.POSTGRES_USERNAME }} + # password: ${{ env.POSTGRES_PASSWORD }} + # database: ${{ env.POSTGRES_DATABASE }} - name: (Windows) Install pgvector using install-pgvector.bat if: matrix.os == 'windows-latest' From 2cdb54a81ff920be4a6f627b7534d25f2ffa8910 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:21:01 +0000 Subject: [PATCH 48/55] Test windows --- .github/workflows/app-tests.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 188ee377..882ff5d0 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -34,7 +34,7 @@ jobs: fail-fast: false matrix: #os: ["ubuntu-latest", "windows-latest", "macos-latest-xlarge", "macos-13"] - os: ["macos-latest-xlarge"] + os: ["windows-latest"] python_version: ["3.11"] #python_version: ["3.10", "3.11", "3.12"] exclude: @@ -54,14 +54,6 @@ jobs: psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'" psql -d postgres -c 'CREATE EXTENSION vector' - # - name: (MacOSLarge) Install postgreSQL using action-setup-postgres - # if: matrix.os == 'macos-latest-xlarge' - # uses: ikalnytskyi/action-setup-postgres@v6 - # with: - # username: ${{ env.POSTGRES_USERNAME }} - # password: ${{ env.POSTGRES_PASSWORD }} - # database: ${{ env.POSTGRES_DATABASE }} - - name: (Windows) Install pgvector using install-pgvector.bat if: matrix.os == 'windows-latest' shell: cmd From 21dd904ee659dae2809ce9d2ae058df026db5f3d Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:27:43 +0000 Subject: [PATCH 49/55] Port to uv --- .github/workflows/app-tests.yaml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 882ff5d0..3c8e0d46 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -24,12 +24,6 @@ jobs: test-package: name: Test ${{ matrix.os }} Python ${{ matrix.python_version }} runs-on: ${{ matrix.os }} - env: - POSTGRES_HOST: localhost - POSTGRES_USERNAME: postgres - POSTGRES_PASSWORD: root - POSTGRES_DATABASE: postgres - POSTGRES_SSL: disable strategy: fail-fast: false matrix: @@ -40,6 +34,13 @@ jobs: exclude: - os: macos-latest-xlarge python_version: "3.10" + env: + UV_SYSTEM_PYTHON: 1 + POSTGRES_HOST: localhost + POSTGRES_USERNAME: postgres + POSTGRES_PASSWORD: root + POSTGRES_DATABASE: postgres + POSTGRES_SSL: disable steps: - uses: actions/checkout@v4 @@ -74,13 +75,20 @@ jobs: python-version: ${{ matrix.python_version }} architecture: x64 + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + version: "0.4.20" + cache-dependency-glob: "requirements**.txt" + - name: Install dependencies run: | - python -m pip install -r requirements-dev.txt + uv pip install -r requirements-dev.txt - name: Install app as editable app run: | - python -m pip install -e src/backend + uv pip install -e src/backend - name: Setup local database with seed data run: | From 951d89b5263a1ef5931ede75218f850a36c85376 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:33:20 +0000 Subject: [PATCH 50/55] Windows fix --- .github/workflows/app-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 3c8e0d46..92274e95 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -37,8 +37,8 @@ jobs: env: UV_SYSTEM_PYTHON: 1 POSTGRES_HOST: localhost - POSTGRES_USERNAME: postgres - POSTGRES_PASSWORD: root + POSTGRES_USERNAME: admin + POSTGRES_PASSWORD: postgres POSTGRES_DATABASE: postgres POSTGRES_SSL: disable steps: From ac53a46a0f31e06a9063919c0b8512fc1b975aed Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:43:10 +0000 Subject: [PATCH 51/55] Start postgres on windows --- .github/workflows/app-tests.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 92274e95..1ed60c69 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -55,6 +55,13 @@ jobs: psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'" psql -d postgres -c 'CREATE EXTENSION vector' + - name: (Windows) Start postgreSQL + uses: ikalnytskyi/action-setup-postgres@v6 + with: + username: ${{ env.POSTGRES_USERNAME }} + password: ${{ env.POSTGRES_PASSWORD }} + database: ${{ env.POSTGRES_DATABASE }} + - name: (Windows) Install pgvector using install-pgvector.bat if: matrix.os == 'windows-latest' shell: cmd From b82e8031be027a4839ad7bf12e8b5fde14f34f14 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:47:25 +0000 Subject: [PATCH 52/55] Start postgres on windows --- .github/workflows/app-tests.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 1ed60c69..61839462 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -55,6 +55,11 @@ jobs: psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'" psql -d postgres -c 'CREATE EXTENSION vector' + - name: (Windows) Install pgvector using install-pgvector.bat + if: matrix.os == 'windows-latest' + shell: cmd + run: .github\workflows\install-pgvector.bat + - name: (Windows) Start postgreSQL uses: ikalnytskyi/action-setup-postgres@v6 with: @@ -62,11 +67,6 @@ jobs: password: ${{ env.POSTGRES_PASSWORD }} database: ${{ env.POSTGRES_DATABASE }} - - name: (Windows) Install pgvector using install-pgvector.bat - if: matrix.os == 'windows-latest' - shell: cmd - run: .github\workflows\install-pgvector.bat - - name: (Linux) Install pgvector and set password if: matrix.os == 'ubuntu-latest' run: | From efd6b753345340949db2984d6132c4a75368973e Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:53:19 +0000 Subject: [PATCH 53/55] Bring back everything but Windows --- .github/workflows/app-tests.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 61839462..6b09b28b 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -27,10 +27,8 @@ jobs: strategy: fail-fast: false matrix: - #os: ["ubuntu-latest", "windows-latest", "macos-latest-xlarge", "macos-13"] - os: ["windows-latest"] - python_version: ["3.11"] - #python_version: ["3.10", "3.11", "3.12"] + os: ["ubuntu-latest", "macos-latest-xlarge", "macos-13"] + python_version: ["3.10", "3.11", "3.12"] exclude: - os: macos-latest-xlarge python_version: "3.10" From 06f25c6307d1d8e53da552ff5f330d0a467a2014 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:55:15 +0000 Subject: [PATCH 54/55] Bring back everything but Windows --- .github/workflows/app-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 6b09b28b..ad197b72 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -59,6 +59,7 @@ jobs: run: .github\workflows\install-pgvector.bat - name: (Windows) Start postgreSQL + if: matrix.os == 'windows-latest' uses: ikalnytskyi/action-setup-postgres@v6 with: username: ${{ env.POSTGRES_USERNAME }} From ea605219c53798c8bc2568a7c6c9e9a0f7729378 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 21:57:15 +0000 Subject: [PATCH 55/55] Dont hardcode password --- .github/workflows/app-tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index ad197b72..066b841d 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -35,8 +35,8 @@ jobs: env: UV_SYSTEM_PYTHON: 1 POSTGRES_HOST: localhost - POSTGRES_USERNAME: admin - POSTGRES_PASSWORD: postgres + POSTGRES_USERNAME: postgres + POSTGRES_PASSWORD: root POSTGRES_DATABASE: postgres POSTGRES_SSL: disable steps: @@ -72,7 +72,7 @@ jobs: sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y sudo apt-get install postgresql-14-pgvector sudo systemctl start postgresql - sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'root'" + sudo -u postgres psql -c "ALTER USER ${{ env.POSTGRES_USERNAME }} PASSWORD '${{ env.POSTGRES_PASSWORD }}'" sudo -u postgres psql -c 'CREATE EXTENSION vector' - name: Setup python