Skip to content

Commit 33b5295

Browse files
committed
Tests better
1 parent 9871f97 commit 33b5295

File tree

5 files changed

+35
-14
lines changed

5 files changed

+35
-14
lines changed

tests/conftest.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import json
12
import os
23
from pathlib import Path
34
from unittest import mock
@@ -253,6 +254,8 @@ async def mock_acreate(*args, **kwargs):
253254
last_question = messages[-1]["content"]
254255
last_role = messages[-1]["role"]
255256
if last_role == "tool":
257+
items = json.loads(last_question)["items"]
258+
arguments = {"query": "capital of France", "items": items, "filters": []}
256259
return ChatCompletion(
257260
object="chat.completion",
258261
choices=[
@@ -265,7 +268,7 @@ async def mock_acreate(*args, **kwargs):
265268
type="function",
266269
function=Function(
267270
name="final_result",
268-
arguments='{"query":"capital of France", "items":[], "filters":[]}',
271+
arguments=json.dumps(arguments),
269272
),
270273
)
271274
],

tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,16 @@
44
"role": "assistant"
55
},
66
"context": {
7-
"data_points": {},
7+
"data_points": {
8+
"1": {
9+
"id": 1,
10+
"type": "Footwear",
11+
"brand": "Daybird",
12+
"name": "Wanderer Black Hiking Boots",
13+
"description": "Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.",
14+
"price": 109.99
15+
}
16+
},
817
"thoughts": [
918
{
1019
"title": "Prompt to generate search arguments",
@@ -87,7 +96,7 @@
8796
"parts": [
8897
{
8998
"content": "Find search results for user query: What is the capital of France?",
90-
"timestamp": "2025-05-07T22:31:16.169127Z",
99+
"timestamp": "2025-05-07T22:53:02.948859Z",
91100
"part_kind": "user-prompt"
92101
}
93102
],
@@ -126,7 +135,7 @@
126135
"filters": []
127136
},
128137
"tool_call_id": "call_abc123",
129-
"timestamp": "2025-05-07T22:31:16.187780Z",
138+
"timestamp": "2025-05-07T22:53:02.963649Z",
130139
"part_kind": "tool-return"
131140
}
132141
],
@@ -137,7 +146,7 @@
137146
"parts": [
138147
{
139148
"tool_name": "final_result",
140-
"args": "{\"query\":\"capital of France\", \"items\":[], \"filters\":[]}",
149+
"args": "{\"query\": \"capital of France\", \"items\": [{\"id\": 1, \"type\": \"Footwear\", \"brand\": \"Daybird\", \"name\": \"Wanderer Black Hiking Boots\", \"description\": \"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.\", \"price\": 109.99}], \"filters\": []}",
141150
"tool_call_id": "call_abc123final",
142151
"part_kind": "tool-call"
143152
}
@@ -152,7 +161,7 @@
152161
"tool_name": "final_result",
153162
"content": "Final result processed.",
154163
"tool_call_id": "call_abc123final",
155-
"timestamp": "2025-05-07T22:31:16.188322Z",
164+
"timestamp": "2025-05-07T22:53:02.964200Z",
156165
"part_kind": "tool-return"
157166
}
158167
],
@@ -177,7 +186,16 @@
177186
},
178187
{
179188
"title": "Search results",
180-
"description": [],
189+
"description": [
190+
{
191+
"id": 1,
192+
"type": "Footwear",
193+
"brand": "Daybird",
194+
"name": "Wanderer Black Hiking Boots",
195+
"description": "Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.",
196+
"price": 109.99
197+
}
198+
],
181199
"props": {}
182200
},
183201
{
@@ -187,13 +205,13 @@
187205
"parts": [
188206
{
189207
"content": "Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51].",
190-
"timestamp": "2025-05-07T22:31:16.188611Z",
208+
"timestamp": "2025-05-07T22:53:02.964504Z",
191209
"dynamic_ref": null,
192210
"part_kind": "system-prompt"
193211
},
194212
{
195-
"content": "What is the capital of France?Sources:\n",
196-
"timestamp": "2025-05-07T22:31:16.188613Z",
213+
"content": "What is the capital of France?Sources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear",
214+
"timestamp": "2025-05-07T22:53:02.964505Z",
197215
"part_kind": "user-prompt"
198216
}
199217
],
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
{"delta":null,"context":{"data_points":{},"thoughts":[{"title":"Prompt to generate search arguments","description":[{"parts":[{"content":"good options for climbing gear that can be used outside?","timestamp":"2025-05-07T19:02:46.977501Z","part_kind":"user-prompt"}],"instructions":null,"kind":"request"},{"parts":[{"tool_name":"search_database","args":"{\"search_query\":\"climbing gear outside\"}","tool_call_id":"call_4HeBCmo2uioV6CyoePEGyZPc","part_kind":"tool-call"}],"model_name":"gpt-4o-mini-2024-07-18","timestamp":"2025-05-07T19:02:47Z","kind":"response"},{"parts":[{"tool_name":"search_database","content":"Search results for climbing gear that can be used outside: ...","tool_call_id":"call_4HeBCmo2uioV6CyoePEGyZPc","timestamp":"2025-05-07T19:02:48.242408Z","part_kind":"tool-return"}],"instructions":null,"kind":"request"},{"parts":[{"content":"are there any shoes less than $50?","timestamp":"2025-05-07T19:02:46.977501Z","part_kind":"user-prompt"}],"instructions":null,"kind":"request"},{"parts":[{"tool_name":"search_database","args":"{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}","tool_call_id":"call_4HeBCmo2uioV6CyoePEGyZPc","part_kind":"tool-call"}],"model_name":"gpt-4o-mini-2024-07-18","timestamp":"2025-05-07T19:02:47Z","kind":"response"},{"parts":[{"tool_name":"search_database","content":"Search results for shoes cheaper than 50: ...","tool_call_id":"call_4HeBCmo2uioV6CyoePEGyZPc","timestamp":"2025-05-07T19:02:48.242408Z","part_kind":"tool-return"}],"instructions":null,"kind":"request"},{"parts":[{"content":"Find search results for user query: What is the capital of France?","timestamp":"2025-05-07T22:31:16.246413Z","part_kind":"user-prompt"}],"instructions":null,"kind":"request"},{"parts":[{"tool_name":"search_database","args":"{\"search_query\":\"climbing gear outside\"}","tool_call_id":"call_abc123","part_kind":"tool-call"}],"model_name":"test-model","timestamp":"1970-01-01T00:00:00Z","kind":"response"},{"parts":[{"tool_name":"search_database","content":{"query":"climbing gear outside","items":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"filters":[]},"tool_call_id":"call_abc123","timestamp":"2025-05-07T22:31:16.268574Z","part_kind":"tool-return"}],"instructions":null,"kind":"request"},{"parts":[{"tool_name":"final_result","args":"{\"query\":\"capital of France\", \"items\":[], \"filters\":[]}","tool_call_id":"call_abc123final","part_kind":"tool-call"}],"model_name":"test-model","timestamp":"1970-01-01T00:00:00Z","kind":"response"},{"parts":[{"tool_name":"final_result","content":"Final result processed.","tool_call_id":"call_abc123final","timestamp":"2025-05-07T22:31:16.269069Z","part_kind":"tool-return"}],"instructions":null,"kind":"request"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}},{"title":"Search using generated search arguments","description":"capital of France","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[],"props":{}},{"title":"Prompt to generate answer","description":[{"parts":[{"content":"Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51].","timestamp":"2025-05-07T22:31:16.269763Z","dynamic_ref":null,"part_kind":"system-prompt"},{"content":"What is the capital of France?Sources:\n","timestamp":"2025-05-07T22:31:16.269765Z","part_kind":"user-prompt"}],"instructions":null,"kind":"request"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}}],"followup_questions":null},"sessionState":null}
1+
{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":[{"parts":[{"content":"good options for climbing gear that can be used outside?","timestamp":"2025-05-07T19:02:46.977501Z","part_kind":"user-prompt"}],"instructions":null,"kind":"request"},{"parts":[{"tool_name":"search_database","args":"{\"search_query\":\"climbing gear outside\"}","tool_call_id":"call_4HeBCmo2uioV6CyoePEGyZPc","part_kind":"tool-call"}],"model_name":"gpt-4o-mini-2024-07-18","timestamp":"2025-05-07T19:02:47Z","kind":"response"},{"parts":[{"tool_name":"search_database","content":"Search results for climbing gear that can be used outside: ...","tool_call_id":"call_4HeBCmo2uioV6CyoePEGyZPc","timestamp":"2025-05-07T19:02:48.242408Z","part_kind":"tool-return"}],"instructions":null,"kind":"request"},{"parts":[{"content":"are there any shoes less than $50?","timestamp":"2025-05-07T19:02:46.977501Z","part_kind":"user-prompt"}],"instructions":null,"kind":"request"},{"parts":[{"tool_name":"search_database","args":"{\"search_query\":\"shoes\",\"price_filter\":{\"comparison_operator\":\"<\",\"value\":50}}","tool_call_id":"call_4HeBCmo2uioV6CyoePEGyZPc","part_kind":"tool-call"}],"model_name":"gpt-4o-mini-2024-07-18","timestamp":"2025-05-07T19:02:47Z","kind":"response"},{"parts":[{"tool_name":"search_database","content":"Search results for shoes cheaper than 50: ...","tool_call_id":"call_4HeBCmo2uioV6CyoePEGyZPc","timestamp":"2025-05-07T19:02:48.242408Z","part_kind":"tool-return"}],"instructions":null,"kind":"request"},{"parts":[{"content":"Find search results for user query: What is the capital of France?","timestamp":"2025-05-07T22:53:03.018949Z","part_kind":"user-prompt"}],"instructions":null,"kind":"request"},{"parts":[{"tool_name":"search_database","args":"{\"search_query\":\"climbing gear outside\"}","tool_call_id":"call_abc123","part_kind":"tool-call"}],"model_name":"test-model","timestamp":"1970-01-01T00:00:00Z","kind":"response"},{"parts":[{"tool_name":"search_database","content":{"query":"climbing gear outside","items":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"filters":[]},"tool_call_id":"call_abc123","timestamp":"2025-05-07T22:53:03.034667Z","part_kind":"tool-return"}],"instructions":null,"kind":"request"},{"parts":[{"tool_name":"final_result","args":"{\"query\": \"capital of France\", \"items\": [{\"id\": 1, \"type\": \"Footwear\", \"brand\": \"Daybird\", \"name\": \"Wanderer Black Hiking Boots\", \"description\": \"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.\", \"price\": 109.99}], \"filters\": []}","tool_call_id":"call_abc123final","part_kind":"tool-call"}],"model_name":"test-model","timestamp":"1970-01-01T00:00:00Z","kind":"response"},{"parts":[{"tool_name":"final_result","content":"Final result processed.","tool_call_id":"call_abc123final","timestamp":"2025-05-07T22:53:03.035160Z","part_kind":"tool-return"}],"instructions":null,"kind":"request"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}},{"title":"Search using generated search arguments","description":"capital of France","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":[{"parts":[{"content":"Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51].","timestamp":"2025-05-07T22:53:03.035786Z","dynamic_ref":null,"part_kind":"system-prompt"},{"content":"What is the capital of France?Sources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear","timestamp":"2025-05-07T22:53:03.035788Z","part_kind":"user-prompt"}],"instructions":null,"kind":"request"}],"props":{"model":"gpt-4o-mini","deployment":"gpt-4o-mini"}}],"followup_questions":null},"sessionState":null}
22
{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null}

tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,13 @@
4545
"parts": [
4646
{
4747
"content": "Assistant helps customers with questions about products.\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\nAnswer ONLY with the product details listed in the products.\nIf there isn't enough information below, say you don't know.\nDo not generate answers that don't use the sources below.\nEach product has an ID in brackets followed by colon and the product details.\nAlways include the product ID for each product you use in the response.\nUse square brackets to reference the source, for example [52].\nDon't combine citations, list each product separately, for example [27][51].",
48-
"timestamp": "2025-05-07T22:31:15.947141Z",
48+
"timestamp": "2025-05-07T22:53:02.797757Z",
4949
"dynamic_ref": null,
5050
"part_kind": "system-prompt"
5151
},
5252
{
5353
"content": "What is the capital of France?Sources:\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear",
54-
"timestamp": "2025-05-07T22:31:15.947145Z",
54+
"timestamp": "2025-05-07T22:53:02.797762Z",
5555
"part_kind": "user-prompt"
5656
}
5757
],

0 commit comments

Comments
 (0)