From b438b6d7cbc7f7c88bc7a5b6bff595661dc5fb4e Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 23:03:36 +0000 Subject: [PATCH 1/6] Test evaluate --- .github/workflows/evaluate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index b0128747..e4e68453 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -1,4 +1,4 @@ -name: Evaluate +name: Evaluate RAG answer quality on: issue_comment: From 882102d0ab8e6125c9022a1c9c86b4063a0401ca Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 23:26:40 +0000 Subject: [PATCH 2/6] Get windows working again --- .github/workflows/app-tests.yaml | 20 +++++++++++++++----- .github/workflows/evaluate.yaml | 10 ++++++---- src/backend/fastapi_app/prompts/query.txt | 2 +- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 066b841d..26a7e83d 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -53,11 +53,6 @@ jobs: psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'" psql -d postgres -c 'CREATE EXTENSION vector' - - name: (Windows) Install pgvector using install-pgvector.bat - if: matrix.os == 'windows-latest' - shell: cmd - run: .github\workflows\install-pgvector.bat - - name: (Windows) Start postgreSQL if: matrix.os == 'windows-latest' uses: ikalnytskyi/action-setup-postgres@v6 @@ -65,6 +60,21 @@ jobs: username: ${{ env.POSTGRES_USERNAME }} password: ${{ env.POSTGRES_PASSWORD }} database: ${{ env.POSTGRES_DATABASE }} + steps: + + - name: Install pgvector + if: matrix.os == 'windows-latest' + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + cd %TEMP% + git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git + cd pgvector + nmake /NOLOGO /F Makefile.win + nmake /NOLOGO /F Makefile.win install + sc config postgresql-x64-14 start=auto + net start postgresql-x64-14 + "%PGBIN%/psql" -d postgres -c "CREATE EXTENSION vector" - name: (Linux) Install pgvector and set password if: matrix.os == 'ubuntu-latest' diff --git a/.github/workflows/evaluate.yaml b/.github/workflows/evaluate.yaml index e4e68453..548db52e 100644 --- a/.github/workflows/evaluate.yaml +++ b/.github/workflows/evaluate.yaml @@ -1,4 +1,4 @@ -name: Evaluate RAG answer quality +name: Evaluate RAG answer flow on: issue_comment: @@ -133,7 +133,7 @@ jobs: - name: Evaluate local RAG flow run: | - python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=results/pr${{ github.event.issue.number }} + python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=evals/results/pr${{ github.event.issue.number }} - name: Upload server logs as build artifact uses: actions/upload-artifact@v4 @@ -163,10 +163,12 @@ jobs: uses: actions/github-script@v7 with: script: | - const { GITHUB_STEP_SUMMARY } = process.env; + const fs = require('fs'); + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + const summary = fs.readFileSync(summaryPath, 'utf8'); github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: GITHUB_STEP_SUMMARY + body: summary }) diff --git a/src/backend/fastapi_app/prompts/query.txt b/src/backend/fastapi_app/prompts/query.txt index 6bbb0a23..d8cdbfc6 100644 --- a/src/backend/fastapi_app/prompts/query.txt +++ b/src/backend/fastapi_app/prompts/query.txt @@ -3,4 +3,4 @@ You have access to an Azure PostgreSQL database with an items table that has col Generate a search query based on the conversation and the new question. If the question is not in English, translate the question to English before generating the search query. If you cannot generate a search query, return the original user question. -DO NOT return anything besides the query. +DO NOT return anything besides the query! From b3bf2e0e172046bbc5c0efe306dc74546f5db205 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 23:36:10 +0000 Subject: [PATCH 3/6] Syntax error --- .github/workflows/app-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 26a7e83d..26c4b24b 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -60,7 +60,6 @@ jobs: username: ${{ env.POSTGRES_USERNAME }} password: ${{ env.POSTGRES_PASSWORD }} database: ${{ env.POSTGRES_DATABASE }} - steps: - name: Install pgvector if: matrix.os == 'windows-latest' From 198059b43648e2de91ecc0bbdd96f15ae171c552 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 23:43:08 +0000 Subject: [PATCH 4/6] Arbitrary comment change to trigger tests --- .github/workflows/app-tests.yaml | 2 +- src/backend/fastapi_app/prompts/query.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 26c4b24b..a7a0a3ab 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -27,7 +27,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "macos-latest-xlarge", "macos-13"] + os: ["ubuntu-latest", "macos-latest-xlarge", "macos-13", "windows-latest"] python_version: ["3.10", "3.11", "3.12"] exclude: - os: macos-latest-xlarge diff --git a/src/backend/fastapi_app/prompts/query.txt b/src/backend/fastapi_app/prompts/query.txt index d8cdbfc6..6bbb0a23 100644 --- a/src/backend/fastapi_app/prompts/query.txt +++ b/src/backend/fastapi_app/prompts/query.txt @@ -3,4 +3,4 @@ You have access to an Azure PostgreSQL database with an items table that has col Generate a search query based on the conversation and the new question. If the question is not in English, translate the question to English before generating the search query. If you cannot generate a search query, return the original user question. -DO NOT return anything besides the query! +DO NOT return anything besides the query. From 32f8aa137ee3826019ba02eae842921860e91da4 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 23:45:55 +0000 Subject: [PATCH 5/6] Arbitrary comment change to trigger tests --- src/backend/fastapi_app/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/fastapi_app/__init__.py b/src/backend/fastapi_app/__init__.py index e4cf5be1..c1be19e5 100644 --- a/src/backend/fastapi_app/__init__.py +++ b/src/backend/fastapi_app/__init__.py @@ -60,7 +60,7 @@ def create_app(testing: bool = False): if os.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"): logger.info("Configuring Azure Monitor") configure_azure_monitor(logger_name="ragapp") - # OpenAI SDK requests use httpx and are thus not auto-instrumented: + # OpenAI SDK requests use httpx, so are thus not auto-instrumented: OpenAIInstrumentor().instrument() app = fastapi.FastAPI(docs_url="/docs", lifespan=lifespan) From 0359ae2001b99684654f706254cf735576641757 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 21 Oct 2024 23:50:00 +0000 Subject: [PATCH 6/6] Remove old windows start --- .github/workflows/app-tests.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index a7a0a3ab..39950b74 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -53,13 +53,13 @@ jobs: psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'" psql -d postgres -c 'CREATE EXTENSION vector' - - name: (Windows) Start postgreSQL - if: matrix.os == 'windows-latest' - uses: ikalnytskyi/action-setup-postgres@v6 - with: - username: ${{ env.POSTGRES_USERNAME }} - password: ${{ env.POSTGRES_PASSWORD }} - database: ${{ env.POSTGRES_DATABASE }} + # - name: (Windows) Start postgreSQL + # if: matrix.os == 'windows-latest' + # uses: ikalnytskyi/action-setup-postgres@v6 + # with: + # username: ${{ env.POSTGRES_USERNAME }} + # password: ${{ env.POSTGRES_PASSWORD }} + # database: ${{ env.POSTGRES_DATABASE }} - name: Install pgvector if: matrix.os == 'windows-latest'