Skip to content

Commit 5b8150a

Browse files
pamelafoxpamelafox
authored
Merge pull request #93 from Azure-Samples/test-eval
Upgrade pgvector and improve eval workflow
2 parents 9392353 + ea60521 commit 5b8150a

21 files changed

+212
-160
lines changed

‎.github/workflows/app-tests.yaml

Lines changed: 57 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -27,73 +27,109 @@ jobs:
2727
strategy:
2828
fail-fast: false
2929
matrix:
30-
os: ["ubuntu-latest", "windows-latest", "macos-latest-xlarge", "macos-13"]
30+
os: ["ubuntu-latest", "macos-latest-xlarge", "macos-13"]
3131
python_version: ["3.10", "3.11", "3.12"]
3232
exclude:
3333
- os: macos-latest-xlarge
3434
python_version: "3.10"
35+
env:
36+
UV_SYSTEM_PYTHON: 1
37+
POSTGRES_HOST: localhost
38+
POSTGRES_USERNAME: postgres
39+
POSTGRES_PASSWORD: root
40+
POSTGRES_DATABASE: postgres
41+
POSTGRES_SSL: disable
3542
steps:
3643
- uses: actions/checkout@v4
37-
- name: Check for MacOS Runner
38-
if: matrix.os == 'macos-latest-xlarge'
39-
run: brew install postgresql@14
40-
- name: Install pgvector on Windows using install-pgvector.bat
44+
45+
- name: (MacOS) Install postgreSQL and pgvector using brew
46+
if: matrix.os == 'macos-13' || matrix.os == 'macos-latest-xlarge'
47+
run: |
48+
brew install postgresql@14
49+
brew link --overwrite postgresql@14
50+
brew install pgvector
51+
brew services start postgresql@14 && sleep 1
52+
createuser -s ${{ env.POSTGRES_USERNAME }}
53+
psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'"
54+
psql -d postgres -c 'CREATE EXTENSION vector'
55+
56+
- name: (Windows) Install pgvector using install-pgvector.bat
4157
if: matrix.os == 'windows-latest'
4258
shell: cmd
4359
run: .github\workflows\install-pgvector.bat
44-
- name: Install PostgreSQL development libraries
45-
if: matrix.os == 'ubuntu-latest'
46-
run: |
47-
sudo apt update
48-
sudo apt install postgresql-server-dev-14
49-
- name: Setup postgres
60+
61+
- name: (Windows) Start postgreSQL
62+
if: matrix.os == 'windows-latest'
5063
uses: ikalnytskyi/action-setup-postgres@v6
5164
with:
52-
username: admin
53-
password: postgres
54-
database: postgres
55-
- name: Install pgvector on MacOS/Linux using install-pgvector.sh
56-
if: matrix.os != 'windows-latest'
57-
run: .github/workflows/install-pgvector.sh
65+
username: ${{ env.POSTGRES_USERNAME }}
66+
password: ${{ env.POSTGRES_PASSWORD }}
67+
database: ${{ env.POSTGRES_DATABASE }}
68+
69+
- name: (Linux) Install pgvector and set password
70+
if: matrix.os == 'ubuntu-latest'
71+
run: |
72+
sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
73+
sudo apt-get install postgresql-14-pgvector
74+
sudo systemctl start postgresql
75+
sudo -u postgres psql -c "ALTER USER ${{ env.POSTGRES_USERNAME }} PASSWORD '${{ env.POSTGRES_PASSWORD }}'"
76+
sudo -u postgres psql -c 'CREATE EXTENSION vector'
77+
5878
- name: Setup python
5979
uses: actions/setup-python@v5
6080
with:
6181
python-version: ${{ matrix.python_version }}
6282
architecture: x64
83+
84+
- name: Install uv
85+
uses: astral-sh/setup-uv@v3
86+
with:
87+
enable-cache: true
88+
version: "0.4.20"
89+
cache-dependency-glob: "requirements**.txt"
90+
6391
- name: Install dependencies
6492
run: |
65-
python -m pip install -r requirements-dev.txt
93+
uv pip install -r requirements-dev.txt
94+
6695
- name: Install app as editable app
6796
run: |
68-
python -m pip install -e src/backend
97+
uv pip install -e src/backend
98+
6999
- name: Setup local database with seed data
70100
run: |
71-
cp .env.sample .env
72101
python ./src/backend/fastapi_app/setup_postgres_database.py
73102
python ./src/backend/fastapi_app/setup_postgres_seeddata.py
103+
74104
- name: Setup node
75105
uses: actions/setup-node@v4
76106
with:
77107
node-version: 18
108+
78109
- name: Build frontend
79110
run: |
80111
cd ./src/frontend
81112
npm install
82113
npm run build
83-
- name: cache mypy
114+
115+
- name: Setup mypy cache
84116
uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
85117
with:
86118
path: ./.mypy_cache
87119
key: mypy${{ matrix.os }}-${{ matrix.python_version }}-${{ hashFiles('requirements-dev.txt', 'src/backend/requirements.txt', 'src/backend/pyproject.toml') }}
120+
88121
- name: Run MyPy
89122
run: python3 -m mypy .
123+
90124
- name: Run Pytest
91125
run: python3 -m pytest -s -vv --cov --cov-fail-under=85
126+
92127
- name: Run E2E tests with Playwright
93128
id: e2e
94129
run: |
95130
playwright install chromium --with-deps
96131
python3 -m pytest tests/e2e.py --tracing=retain-on-failure
132+
97133
- name: Upload test artifacts
98134
if: ${{ failure() && steps.e2e.conclusion == 'failure' }}
99135
uses: actions/upload-artifact@v4

‎.github/workflows/evaluate.yaml

Lines changed: 71 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
name: Evaluate
22

33
on:
4-
workflow_dispatch:
54
issue_comment:
65
types: [created]
76

@@ -13,77 +12,68 @@ permissions:
1312

1413
jobs:
1514
evaluate:
16-
if: github.event_name == 'workflow_dispatch' || contains(github.event.comment.body, '#evaluate')
15+
if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '#evaluate')
1716
runs-on: ubuntu-latest
1817
env:
18+
UV_SYSTEM_PYTHON: 1
1919
AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }}
2020
AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}
2121
AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}
2222
AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
2323
AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
24+
POSTGRES_HOST: localhost
25+
POSTGRES_USERNAME: postgres
26+
POSTGRES_PASSWORD: root
27+
POSTGRES_DATABASE: postgres
28+
POSTGRES_SSL: disable
29+
OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }}
30+
OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }}
31+
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
32+
AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }}
33+
AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }}
34+
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }}
35+
AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }}
36+
AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }}
37+
AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }}
38+
AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }}
39+
AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }}
40+
AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }}
2441
steps:
2542
- name: Check for evaluate hash tag
2643
if: contains(github.event.comment.body, '#evaluate')
2744
run: |
2845
echo "Comment contains #evaluate hashtag"
2946
3047
- uses: actions/checkout@v4
31-
- name: Install PostgreSQL development libraries
48+
49+
- name: Install pgvector
3250
run: |
33-
sudo apt update
34-
sudo apt install postgresql-server-dev-14
35-
- name: Setup postgres
36-
uses: ikalnytskyi/action-setup-postgres@v6
37-
with:
38-
username: admin
39-
password: postgres
40-
database: postgres
51+
sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
52+
sudo apt-get install postgresql-14-pgvector
53+
54+
- name: Start postgres
55+
run: sudo systemctl start postgresql
56+
57+
- name: Set password for postgres user
58+
run: sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'root'"
4159

42-
- name: Install pgvector on MacOS/Linux using install-pgvector.sh
43-
run: .github/workflows/install-pgvector.sh
60+
- name: Create vector extension
61+
run: sudo -u postgres psql -c 'CREATE EXTENSION vector'
4462

4563
- name: Install python
4664
uses: actions/setup-python@v5
4765
with:
4866
python-version: '3.12'
4967

50-
- name: Install azd
51-
uses: Azure/[email protected]
52-
53-
- name: Install dependencies
54-
run: |
55-
python -m pip install -r requirements-dev.txt
56-
57-
- name: Install app as editable app
58-
run: |
59-
python -m pip install -e src/backend
60-
61-
- name: Setup local database with seed data
62-
run: |
63-
python ./src/backend/fastapi_app/setup_postgres_database.py
64-
python ./src/backend/fastapi_app/setup_postgres_seeddata.py
65-
env:
66-
POSTGRES_HOST: localhost
67-
POSTGRES_USERNAME: admin
68-
POSTGRES_PASSWORD: postgres
69-
POSTGRES_DATABASE: postgres
70-
POSTGRES_SSL: disable
71-
72-
- name: Setup node
73-
uses: actions/setup-node@v4
68+
- name: Install uv
69+
uses: astral-sh/setup-uv@v3
7470
with:
75-
node-version: 18
71+
enable-cache: true
72+
version: "0.4.20"
73+
cache-dependency-glob: "requirements**.txt"
7674

77-
- name: Build frontend
78-
run: |
79-
cd ./src/frontend
80-
npm install
81-
npm run build
82-
83-
- name: Install python packages
84-
run: |
85-
python -m pip install --upgrade pip
86-
pip install -r requirements-dev.txt
75+
- name: Install azd
76+
uses: Azure/[email protected]
8777

8878
- name: Login to Azure
8979
uses: azure/login@v2
@@ -107,41 +97,42 @@ jobs:
10797
--tenant-id "$Env:AZURE_TENANT_ID"
10898
shell: pwsh
10999

110-
- name: Provision Infrastructure
111-
run: azd provision --no-prompt
112-
env:
113-
AZD_INITIAL_ENVIRONMENT_CONFIG: ${{ secrets.AZD_INITIAL_ENVIRONMENT_CONFIG }}
100+
- name: Install dependencies
101+
run: |
102+
uv pip install -r requirements-dev.txt
103+
104+
- name: Install app as editable app
105+
run: |
106+
uv pip install -e src/backend
107+
108+
- name: Setup local database with seed data
109+
run: |
110+
python ./src/backend/fastapi_app/setup_postgres_database.py
111+
python ./src/backend/fastapi_app/setup_postgres_seeddata.py
112+
113+
- name: Setup node
114+
uses: actions/setup-node@v4
115+
with:
116+
node-version: 18
117+
118+
- name: Build frontend
119+
run: |
120+
cd ./src/frontend
121+
npm install
122+
npm run build
114123
115124
- name: Run local server in background
116125
run: |
117126
RUNNER_TRACKING_ID="" && (nohup python3 -m uvicorn fastapi_app:create_app --factory > serverlogs.out 2> serverlogs.err &)
118-
env:
119-
OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }}
120-
OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }}
121-
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
122-
AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }}
123-
AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }}
124-
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }}
125-
AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }}
126-
AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }}
127-
AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }}
128-
AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }}
129-
POSTGRES_HOST: localhost
130-
POSTGRES_USERNAME: admin
131-
POSTGRES_PASSWORD: postgres
132-
POSTGRES_DATABASE: postgres
133-
POSTGRES_SSL: disable
127+
128+
- name: Install evaluate dependencies
129+
run: |
130+
uv pip install -r evals/requirements.txt
131+
134132
- name: Evaluate local RAG flow
135133
run: |
136-
python evals/evaluate.py
137-
env:
138-
OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }}
139-
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
140-
AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }}
141-
AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }}
142-
AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }}
143-
AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }}
144-
AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }}
134+
python evals/evaluate.py --targeturl=http://127.0.0.1:8000/chat --numquestions=2 --resultsdir=results/pr${{ github.event.issue.number }}
135+
145136
- name: Upload server logs as build artifact
146137
uses: actions/upload-artifact@v4
147138
with:
@@ -158,13 +149,10 @@ jobs:
158149
uses: actions/upload-artifact@v4
159150
with:
160151
name: eval_result
161-
path: ./src/api/evaluate/eval_results.jsonl
152+
path: ./evals/results/pr${{ github.event.issue.number }}
162153

163154
- name: GitHub Summary Step
164155
if: ${{ success() }}
165-
working-directory: ./src/api
166156
run: |
167-
echo "" >> $GITHUB_STEP_SUMMARY
168-
169-
echo "📊 Promptflow Evaluation Results" >> $GITHUB_STEP_SUMMARY
170-
cat evaluate/eval_results.md >> $GITHUB_STEP_SUMMARY
157+
echo "📊 Evaluation Results" >> $GITHUB_STEP_SUMMARY
158+
python -m evaltools summary evals/results --output=markdown >> $GITHUB_STEP_SUMMARY

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ Since the local app uses OpenAI models, you should first deploy it for the optim
139139
```
140140
141141
3. To use OpenAI.com OpenAI, set `OPENAI_CHAT_HOST` and `OPENAI_EMBED_HOST` to "openai". Then fill in the value for `OPENAICOM_KEY`.
142-
4. To use Ollama, set `OPENAI_CHAT_HOST` to "ollama". Then update the values for `OLLAMA_ENDPOINT` and `OLLAMA_CHAT_MODEL` to match your local setup and model. Note that most Ollama models are not compatible with the "Advanced flow", due to the need for function calling support, so you'll need to disable that in _Developer Settings_ in the UI. In addition, the database rows are embedded using the default OpenAI embedding model, so you can't search them using an Ollama embedding model. You can either choose to set `OPENAI_EMBED_HOST` to "azure" or "openai", or turn off vector search in _Developer Settings_.
142+
4. To use Ollama, set `OPENAI_CHAT_HOST` to "ollama". Then update the values for `OLLAMA_ENDPOINT` and `OLLAMA_CHAT_MODEL` to match your local setup and model. We recommend using "llama3.1" for the chat model, since it has support for function calling, and "nomic-embed-text" for the embedding model, since the sample data has already been embedded with this model. If you cannot use function calling, then turn off "Advanced flow" in the Developer Settings. If you cannot use the embedding model, then turn off vector search in the Developer Settings.
143143
144144
### Running the frontend and backend
145145

‎evals/evaluate.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import argparse
12
import logging
23
import os
34
from pathlib import Path
@@ -50,11 +51,20 @@ def get_openai_config() -> dict:
5051
)
5152
load_dotenv(".env", override=True)
5253

54+
parser = argparse.ArgumentParser(description="Run evaluation with OpenAI configuration.")
55+
parser.add_argument("--targeturl", type=str, help="Specify the target URL.")
56+
parser.add_argument("--resultsdir", type=Path, help="Specify the results directory.")
57+
parser.add_argument("--numquestions", type=int, help="Specify the number of questions.")
58+
59+
args = parser.parse_args()
60+
5361
openai_config = get_openai_config()
54-
# TODO: specify the localhost URL using argument
55-
# TODO: specify the experiment name (based on PR number)
56-
# TODO: Specify the num questions using argument
5762

5863
run_evaluate_from_config(
59-
working_dir=Path(__file__).parent, config_path="eval_config.json", openai_config=openai_config, num_questions=2
64+
working_dir=Path(__file__).parent,
65+
config_path="eval_config.json",
66+
num_questions=args.numquestions,
67+
target_url=args.targeturl,
68+
results_dir=args.resultsdir,
69+
openai_config=openai_config,
6070
)

‎evals/requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
git+https://github.com/Azure-Samples/ai-rag-chat-evaluator/@installable

‎infra/main.bicep

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -449,7 +449,7 @@ output AZURE_OPENAI_RESOURCE_GROUP string = deployAzureOpenAI ? openAIResourceGr
449449
output AZURE_OPENAI_ENDPOINT string = !empty(azureOpenAIEndpoint)
450450
? azureOpenAIEndpoint
451451
: (deployAzureOpenAI ? openAI.outputs.endpoint : '')
452-
output AZURE_OPENAI_VERSION string = openAIEmbedHost == 'chat' ? azureOpenAIAPIVersion : ''
452+
output AZURE_OPENAI_VERSION string = azureOpenAIAPIVersion
453453
output AZURE_OPENAI_CHAT_DEPLOYMENT string = deployAzureOpenAI ? chatDeploymentName : ''
454454
output AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION string = deployAzureOpenAI ? chatDeploymentVersion : ''
455455
output AZURE_OPENAI_CHAT_DEPLOYMENT_CAPACITY int = deployAzureOpenAI ? chatDeploymentCapacity : 0

0 commit comments

Comments
 (0)