Skip to content

Commit 91c0096

Browse files
nina-kollmannina-kollman
committed
added tests
1 parent a8719d8 commit 91c0096

File tree

3 files changed

+496
-16
lines changed

3 files changed

+496
-16
lines changed

‎packages/traceloop-sdk/pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ pandas = ">=1.0.0"
9898
[tool.poetry.extras]
9999
datasets = ["pandas"]
100100

101+
[tool.pytest.ini_options]
102+
asyncio_default_fixture_loop_scope = "function"
103+
101104
[tool.mypy]
102105
python_version = "3.10"
103106
warn_return_any = true

‎packages/traceloop-sdk/tests/evaluator/test_evaluator.py‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
import pytest
2+
from traceloop.sdk.evaluator.evaluator import validate_task_output
3+
from traceloop.sdk.evaluator.config import EvaluatorDetails
4+
5+
6+
class TestValidateTaskOutput:
7+
"""Tests for validate_task_output function"""
8+
9+
def test_validate_task_output_with_no_evaluators(self):
10+
"""Test that validation passes when no evaluators are provided"""
11+
task_output = {"text": "hello"}
12+
evaluators = []
13+
14+
# Should not raise any exception
15+
validate_task_output(task_output, evaluators)
16+
17+
def test_validate_task_output_with_evaluators_no_required_fields(self):
18+
"""Test that validation passes when evaluators have no required fields"""
19+
task_output = {"text": "hello", "score": 0.9}
20+
evaluators = [
21+
EvaluatorDetails(slug="evaluator1"),
22+
EvaluatorDetails(slug="evaluator2", config={"threshold": 0.5}),
23+
]
24+
25+
# Should not raise any exception
26+
validate_task_output(task_output, evaluators)
27+
28+
def test_validate_task_output_with_valid_output(self):
29+
"""Test that validation passes when all required fields are present"""
30+
task_output = {"text": "hello", "prompt": "say hello", "response": "world"}
31+
evaluators = [
32+
EvaluatorDetails(slug="pii-detector", required_input_fields=["text"]),
33+
EvaluatorDetails(
34+
slug="relevance-checker",
35+
required_input_fields=["prompt", "response"],
36+
),
37+
]
38+
39+
# Should not raise any exception
40+
validate_task_output(task_output, evaluators)
41+
42+
def test_validate_task_output_missing_single_field(self):
43+
"""Test that validation fails when a single required field is missing"""
44+
task_output = {"prompt": "say hello"}
45+
evaluators = [
46+
EvaluatorDetails(slug="pii-detector", required_input_fields=["text"]),
47+
]
48+
49+
with pytest.raises(ValueError) as exc_info:
50+
validate_task_output(task_output, evaluators)
51+
52+
error_message = str(exc_info.value)
53+
assert "Task output missing required fields for evaluators:" in error_message
54+
assert "pii-detector requires: ['text']" in error_message
55+
assert "Task output contains: ['prompt']" in error_message
56+
assert (
57+
"Hint: Update your task function to return a dictionary "
58+
"with the required fields."
59+
) in error_message
60+
61+
def test_validate_task_output_missing_multiple_fields_single_evaluator(self):
62+
"""Test that validation fails when multiple fields are missing for one evaluator"""
63+
task_output = {"score": 0.9}
64+
evaluators = [
65+
EvaluatorDetails(
66+
slug="relevance-checker",
67+
required_input_fields=["prompt", "response", "context"],
68+
),
69+
]
70+
71+
with pytest.raises(ValueError) as exc_info:
72+
validate_task_output(task_output, evaluators)
73+
74+
error_message = str(exc_info.value)
75+
assert "relevance-checker requires:" in error_message
76+
assert "'context'" in error_message
77+
assert "'prompt'" in error_message
78+
assert "'response'" in error_message
79+
80+
def test_validate_task_output_missing_fields_multiple_evaluators(self):
81+
"""Test that validation fails when fields are missing for multiple evaluators"""
82+
task_output = {"score": 0.9}
83+
evaluators = [
84+
EvaluatorDetails(slug="pii-detector", required_input_fields=["text"]),
85+
EvaluatorDetails(
86+
slug="relevance-checker",
87+
required_input_fields=["prompt", "response"],
88+
),
89+
EvaluatorDetails(slug="tone-analyzer", required_input_fields=["text"]),
90+
]
91+
92+
with pytest.raises(ValueError) as exc_info:
93+
validate_task_output(task_output, evaluators)
94+
95+
error_message = str(exc_info.value)
96+
assert "pii-detector requires:" in error_message
97+
assert "relevance-checker requires:" in error_message
98+
assert "tone-analyzer requires:" in error_message
99+
assert "'text'" in error_message
100+
assert "'prompt'" in error_message
101+
assert "'response'" in error_message
102+
103+
def test_validate_task_output_partial_match(self):
104+
"""Test validation when some evaluators pass and some fail"""
105+
task_output = {"text": "hello world"}
106+
evaluators = [
107+
EvaluatorDetails(slug="pii-detector", required_input_fields=["text"]),
108+
EvaluatorDetails(
109+
slug="relevance-checker",
110+
required_input_fields=["prompt", "response"],
111+
),
112+
]
113+
114+
with pytest.raises(ValueError) as exc_info:
115+
validate_task_output(task_output, evaluators)
116+
117+
error_message = str(exc_info.value)
118+
# Should only mention the failing evaluator
119+
assert "relevance-checker requires:" in error_message
120+
assert "pii-detector" not in error_message or "pii-detector requires:" not in error_message
121+
122+
def test_validate_task_output_empty_task_output(self):
123+
"""Test validation with empty task output"""
124+
task_output = {}
125+
evaluators = [
126+
EvaluatorDetails(slug="pii-detector", required_input_fields=["text"]),
127+
]
128+
129+
with pytest.raises(ValueError) as exc_info:
130+
validate_task_output(task_output, evaluators)
131+
132+
error_message = str(exc_info.value)
133+
assert "Task output contains: []" in error_message
134+
135+
def test_validate_task_output_with_extra_fields(self):
136+
"""Test that validation passes when task output has extra fields"""
137+
task_output = {
138+
"text": "hello",
139+
"prompt": "say hello",
140+
"response": "world",
141+
"extra_field": "value",
142+
"another_field": 123,
143+
}
144+
evaluators = [
145+
EvaluatorDetails(slug="pii-detector", required_input_fields=["text"]),
146+
]
147+
148+
# Should not raise any exception - extra fields are allowed
149+
validate_task_output(task_output, evaluators)
150+
151+
def test_validate_task_output_case_sensitive_field_names(self):
152+
"""Test that field name matching is case-sensitive"""
153+
task_output = {"Text": "hello"}
154+
evaluators = [
155+
EvaluatorDetails(slug="pii-detector", required_input_fields=["text"]),
156+
]
157+
158+
with pytest.raises(ValueError) as exc_info:
159+
validate_task_output(task_output, evaluators)
160+
161+
error_message = str(exc_info.value)
162+
assert "pii-detector requires: ['text']" in error_message
163+
assert "Task output contains: ['Text']" in error_message
164+
165+
def test_validate_task_output_with_evaluator_config(self):
166+
"""Test validation with evaluators that have config"""
167+
task_output = {"text": "hello world"}
168+
evaluators = [
169+
EvaluatorDetails(
170+
slug="pii-detector",
171+
version="v2",
172+
config={"probability_threshold": 0.8, "mode": "strict"},
173+
required_input_fields=["text"],
174+
),
175+
]
176+
177+
# Should not raise any exception - config shouldn't affect validation
178+
validate_task_output(task_output, evaluators)
179+
180+
def test_validate_task_output_mixed_evaluators(self):
181+
"""Test validation with a mix of evaluators with and without required fields"""
182+
task_output = {"text": "hello", "score": 0.9}
183+
evaluators = [
184+
EvaluatorDetails(slug="evaluator-no-requirements"),
185+
EvaluatorDetails(slug="pii-detector", required_input_fields=["text"]),
186+
EvaluatorDetails(slug="another-no-requirements", config={"key": "value"}),
187+
EvaluatorDetails(
188+
slug="relevance-checker",
189+
required_input_fields=["prompt"],
190+
),
191+
]
192+
193+
with pytest.raises(ValueError) as exc_info:
194+
validate_task_output(task_output, evaluators)
195+
196+
error_message = str(exc_info.value)
197+
# Should only mention failing evaluator
198+
assert "relevance-checker requires: ['prompt']" in error_message
199+
assert "evaluator-no-requirements" not in error_message
200+
assert "pii-detector" not in error_message or "pii-detector requires:" not in error_message
201+
202+
def test_validate_task_output_duplicate_required_fields(self):
203+
"""Test validation when multiple evaluators require the same field"""
204+
task_output = {"score": 0.9}
205+
evaluators = [
206+
EvaluatorDetails(slug="pii-detector", required_input_fields=["text"]),
207+
EvaluatorDetails(slug="tone-analyzer", required_input_fields=["text"]),
208+
EvaluatorDetails(
209+
slug="sentiment-analyzer",
210+
required_input_fields=["text", "language"],
211+
),
212+
]
213+
214+
with pytest.raises(ValueError) as exc_info:
215+
validate_task_output(task_output, evaluators)
216+
217+
error_message = str(exc_info.value)
218+
assert "pii-detector requires:" in error_message
219+
assert "tone-analyzer requires:" in error_message
220+
assert "sentiment-analyzer requires:" in error_message

0 commit comments

Comments
 (0)