diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index cd408336..552bc0e7 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -27,6 +27,9 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: postgres + ports: + - "5432:5432" + # For local developemnt, we need to forward the database port here too. # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index ec3e5e1b..c1c14bd1 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -73,4 +73,16 @@ jobs: - name: Run MyPy run: python3 -m mypy . - name: Run Pytest - run: python3 -m pytest + run: python3 -m pytest -s -vv --cov --cov-fail-under=85 + - name: Run E2E tests with Playwright + id: e2e + if: runner.os != 'Windows' + run: | + playwright install chromium --with-deps + python3 -m pytest tests/e2e.py --tracing=retain-on-failure + - name: Upload test artifacts + if: ${{ failure() && steps.e2e.conclusion == 'failure' }} + uses: actions/upload-artifact@v4 + with: + name: playwright-traces${{ matrix.python_version }} + path: test-results diff --git a/.gitignore b/.gitignore index 20dda899..18609cf6 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,5 @@ npm-debug.log* node_modules static/ +# Playwright test trace +test-results/ diff --git a/pyproject.toml b/pyproject.toml index 5905aa19..1f168be0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,9 @@ python_version = 3.12 exclude = [".venv/*"] [tool.pytest.ini_options] -addopts = "-ra --cov" +addopts = "-ra" testpaths = ["tests"] -pythonpath = ['src'] +pythonpath = ['src/backend'] filterwarnings = ["ignore::DeprecationWarning"] [[tool.mypy.overrides]] diff --git a/requirements-dev.txt b/requirements-dev.txt index 1d7ad271..08ce71aa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,12 +1,14 @@ -r src/backend/requirements.txt ruff mypy +types-requests pre-commit pip-tools pip-compile-cross-platform +playwright pytest -pytest-cov pytest-asyncio +pytest-cov +pytest-playwright pytest-snapshot -mypy locust diff --git a/src/backend/fastapi_app/api_models.py b/src/backend/fastapi_app/api_models.py index c98ca76d..616275c2 100644 --- a/src/backend/fastapi_app/api_models.py +++ b/src/backend/fastapi_app/api_models.py @@ -37,6 +37,7 @@ class ChatRequestContext(BaseModel): class ChatRequest(BaseModel): messages: list[ChatCompletionMessageParam] context: ChatRequestContext + sessionState: Any | None = None class ThoughtStep(BaseModel): @@ -54,13 +55,13 @@ class RAGContext(BaseModel): class RetrievalResponse(BaseModel): message: Message context: RAGContext - session_state: Any | None = None + sessionState: Any | None = None class RetrievalResponseDelta(BaseModel): delta: Message | None = None context: RAGContext | None = None - session_state: Any | None = None + sessionState: Any | None = None class ItemPublic(BaseModel): diff --git a/src/frontend/src/pages/chat/Chat.tsx b/src/frontend/src/pages/chat/Chat.tsx index da0b6934..f583f012 100644 --- a/src/frontend/src/pages/chat/Chat.tsx +++ b/src/frontend/src/pages/chat/Chat.tsx @@ -108,7 +108,8 @@ const Chat = () => { prompt_template: promptTemplate.length === 0 ? undefined : promptTemplate, temperature: temperature } - } + }, + sessionState: answers.length ? answers[answers.length - 1][1].sessionState : null }; const chatClient: AIChatProtocolClient = new AIChatProtocolClient("/chat"); if (shouldStream) { diff --git a/tests/e2e.py b/tests/e2e.py new file mode 100644 index 00000000..ace7f591 --- /dev/null +++ b/tests/e2e.py @@ -0,0 +1,192 @@ +import socket +import time +from collections.abc import Generator +from contextlib import closing +from multiprocessing import Process + +import pytest +import requests +import uvicorn +from playwright.sync_api import Page, Route, expect + +import fastapi_app as app + +expect.set_options(timeout=10_000) + + +def wait_for_server_ready(url: str, timeout: float = 10.0, check_interval: float = 0.5) -> bool: + """Make requests to provided url until it responds without error.""" + conn_error = None + for _ in range(int(timeout / check_interval)): + try: + requests.get(url) + except requests.ConnectionError as exc: + time.sleep(check_interval) + conn_error = str(exc) + else: + return True + raise RuntimeError(conn_error) + + +@pytest.fixture(scope="session") +def free_port() -> int: + """Returns a free port for the test server to bind.""" + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + +def run_server(port: int): + uvicorn.run(app.create_app(testing=True), port=port) + + +@pytest.fixture() +def live_server_url(mock_session_env, free_port: int) -> Generator[str, None, None]: + proc = Process(target=run_server, args=(free_port,), daemon=True) + proc.start() + url = f"http://localhost:{free_port}/" + wait_for_server_ready(url, timeout=10.0, check_interval=0.5) + yield url + proc.kill() + + +@pytest.fixture(params=[(480, 800), (600, 1024), (768, 1024), (992, 1024), (1024, 768)]) +def sized_page(page: Page, request): + size = request.param + page.set_viewport_size({"width": size[0], "height": size[1]}) + yield page + + +def test_home(page: Page, live_server_url: str): + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + + +def test_chat(sized_page: Page, live_server_url: str): + page = sized_page + + # Set up a mock route to the /chat endpoint with streaming results + def handle(route: Route): + # Assert that session_state is specified in the request (None for now) + if route.request.post_data_json: + session_state = route.request.post_data_json["sessionState"] + assert session_state is None + # Read the JSONL from our snapshot results and return as the response + f = open( + "tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines" + ) + jsonl = f.read() + f.close() + route.fulfill(body=jsonl, status=200, headers={"Transfer-encoding": "Chunked"}) + + page.route("*/**/chat/stream", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + expect(page.get_by_role("heading", name="Product chat")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_role("button", name="Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() + + # Show the thought process + page.get_by_label("Show thought process").click() + expect(page.get_by_title("Thought process")).to_be_visible() + expect(page.get_by_text("Prompt to generate search arguments")).to_be_visible() + + # Clear the chat + page.get_by_role("button", name="Clear chat").click() + expect(page.get_by_text("Whats the dental plan?")).not_to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).not_to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() + + +def test_chat_customization(page: Page, live_server_url: str): + # Set up a mock route to the /chat endpoint + def handle(route: Route): + if route.request.post_data_json: + overrides = route.request.post_data_json["context"]["overrides"] + assert overrides["use_advanced_flow"] is False + assert overrides["retrieval_mode"] == "vectors" + assert overrides["top"] == 1 + assert overrides["prompt_template"] == "You are a cat and only talk about tuna." + + # Read the JSON from our snapshot results and return as the response + f = open("tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json") + json = f.read() + f.close() + route.fulfill(body=json, status=200) + + page.route("*/**/chat", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + + # Customize all the settings + page.get_by_role("button", name="Developer settings").click() + page.get_by_text( + "Use advanced flow with query rewriting and filter formulation. Not compatible with Ollama models." + ).click() + page.get_by_label("Retrieve this many matching rows:").click() + page.get_by_label("Retrieve this many matching rows:").fill("1") + page.get_by_text("Vectors + Text (Hybrid)").click() + page.get_by_role("option", name="Vectors", exact=True).click() + page.get_by_label("Override prompt template").click() + page.get_by_label("Override prompt template").fill("You are a cat and only talk about tuna.") + + page.get_by_text("Stream chat completion responses").click() + page.locator("button").filter(has_text="Close").click() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_role("button", name="Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() + + +def test_chat_nonstreaming(page: Page, live_server_url: str): + # Set up a mock route to the /chat_stream endpoint + def handle(route: Route): + # Read the JSON from our snapshot results and return as the response + f = open("tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json") + json = f.read() + f.close() + route.fulfill(body=json, status=200) + + page.route("*/**/chat", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() + page.get_by_role("button", name="Developer settings").click() + page.get_by_text("Stream chat completion responses").click() + page.locator("button").filter(has_text="Close").click() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_label("Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json index 2e9eb3ae..c7692bd1 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json @@ -64,5 +64,5 @@ ], "followup_questions": null }, - "session_state": null + "sessionState": null } \ No newline at end of file diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines index 8b65342f..b7e4efa3 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"session_state":null} -{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"session_state":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} diff --git a/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json b/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json index d5ecba21..a73ff24d 100644 --- a/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json @@ -52,5 +52,5 @@ ], "followup_questions": null }, - "session_state": null + "sessionState": null } \ No newline at end of file diff --git a/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines index 6251bd52..fc63aea9 100644 --- a/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Search query for database","description":"What is the capital of France?","props":{"top":1,"vector_search":true,"text_search":true}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"session_state":null} -{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"session_state":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Search query for database","description":"What is the capital of France?","props":{"top":1,"vector_search":true,"text_search":true}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null}