Repository navigation
Expand file tree
/
Copy pathtest_evaluator.py
More file actions
353 lines (266 loc) · 12.6 KB
/
Copy pathtest_evaluator.py
File metadata and controls
353 lines (266 loc) · 12.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
"""Tests for the evaluator implementations."""
from unittest.mock import Mock
from coder_eval.evaluation.checker import SuccessChecker
from coder_eval.models import (
FileContainsCriterion,
FileExistsCriterion,
FileMatchesRegexCriterion,
RunCommandCriterion,
SandboxConfig,
)
from coder_eval.sandbox import Sandbox
def test_success_checker_file_exists():
"""Test file existence checking."""
config = SandboxConfig(driver="tempdir", python=None)
sandbox = Sandbox(config, task_id="test_eval")
try:
sandbox_dir = sandbox.setup()
# Create a test file
test_file = sandbox_dir / "test.txt"
test_file.write_text("Hello")
# Check file that exists - use SuccessChecker.check()
checker = SuccessChecker(sandbox)
criterion = FileExistsCriterion(path="test.txt", description="Test file should exist")
result = checker.check(criterion)
assert result.score == 1.0
# Check file that doesn't exist
criterion = FileExistsCriterion(path="missing.txt", description="Missing file")
result = checker.check(criterion)
assert result.score == 0.0
finally:
sandbox.cleanup()
def test_success_checker_populates_pass_threshold_on_results():
"""Every CriterionResult returned by SuccessChecker carries the criterion's pass_threshold."""
config = SandboxConfig(driver="tempdir", python=None)
sandbox = Sandbox(config, task_id="test_eval_threshold")
try:
sandbox.setup()
checker = SuccessChecker(sandbox)
# Custom pass_threshold — result must reflect it
criterion = FileExistsCriterion(path="missing.txt", description="missing", pass_threshold=0.5)
result = checker.check(criterion)
assert result.pass_threshold == 0.5
# Unsupported type hits the KeyError branch — still carries the threshold
class _Fake:
type = "unsupported_type"
description = "fake"
pass_threshold = 0.42
is_gating = True
result = checker._check_single(_Fake()) # type: ignore[arg-type]
assert result.pass_threshold == 0.42
assert result.score == 0.0
finally:
sandbox.cleanup()
def test_success_checker_file_contains():
"""Test file content checking."""
config = SandboxConfig(driver="tempdir", python=None)
sandbox = Sandbox(config, task_id="test_eval_contains")
try:
sandbox_dir = sandbox.setup()
# Create a test file with content
test_file = sandbox_dir / "app.py"
test_file.write_text("import datetime\nprint('Hello, Claude!')")
# Check file contains required strings - use SuccessChecker.check()
checker = SuccessChecker(sandbox)
criterion = FileContainsCriterion(
path="app.py", includes=["Hello, Claude!", "datetime"], description="File should contain required strings"
)
result = checker.check(criterion)
assert result.score == 1.0
# Check file is missing required strings
criterion = FileContainsCriterion(
path="app.py", includes=["missing_string"], description="File should contain missing string"
)
result = checker.check(criterion)
# includes: 0/1=0.0, no excludes so score = includes_score only
assert result.score == 0.0
# Check file contains excluded strings
criterion = FileContainsCriterion(
path="app.py",
includes=["Hello"],
excludes=["datetime"], # This IS in the file
description="File should not contain datetime",
)
result = checker.check(criterion)
# includes: 1/1=1.0, excludes: 0/1=0.0, avg=(1.0+0.0)/2=0.5
assert result.score == 0.5
finally:
sandbox.cleanup()
def test_success_checker_run_command():
"""Test command execution checking."""
config = SandboxConfig(driver="tempdir", python=None)
sandbox = Sandbox(config, task_id="test_eval_cmd")
try:
sandbox.setup()
# Test successful command - use SuccessChecker.check()
checker = SuccessChecker(sandbox)
criterion = RunCommandCriterion(
command="echo 'Hello'", timeout=5, expected_exit_code=0, description="Echo should succeed"
)
result = checker.check(criterion)
assert result.score == 1.0
assert "Hello" in result.details
# Test failing command
criterion = RunCommandCriterion(
command="exit 1", timeout=5, expected_exit_code=0, description="This should fail"
)
result = checker.check(criterion)
assert result.score < 1.0
finally:
sandbox.cleanup()
def test_success_checker_check_all():
"""Test checking multiple criteria."""
config = SandboxConfig(driver="tempdir", python=None)
sandbox = Sandbox(config, task_id="test_eval_all")
try:
sandbox_dir = sandbox.setup()
# Create test files
(sandbox_dir / "file1.txt").write_text("content1")
(sandbox_dir / "file2.txt").write_text("content2")
checker = SuccessChecker(sandbox)
criteria = [
FileExistsCriterion(path="file1.txt", description="File 1 exists"),
FileExistsCriterion(path="file2.txt", description="File 2 exists"),
FileExistsCriterion(path="missing.txt", description="Missing file"),
]
results = checker.check_all(criteria)
assert len(results) == 3
assert results[0].score == 1.0
assert results[1].score == 1.0
assert results[2].score == 0.0
finally:
sandbox.cleanup()
def test_success_checker_dispatch():
"""Test pattern matching dispatcher works for all criterion types."""
mock_sandbox = Mock()
mock_sandbox.file_exists.return_value = True
mock_sandbox.get_file_content.return_value = "test content"
mock_sandbox.run_command.return_value = (0, "output", "")
checker = SuccessChecker(mock_sandbox)
# Test each criterion type dispatches correctly
criteria = [
FileExistsCriterion(path="test.txt", description="Test file exists"),
FileContainsCriterion(path="app.py", includes=["test"], description="Test file contains"),
RunCommandCriterion(command="echo test", description="Test run command"),
RunCommandCriterion(command="echo test", expected_stdout="test", description="Test stdout match"),
FileMatchesRegexCriterion(path="app.py", pattern="test", description="Test regex"),
]
for criterion in criteria:
result = checker.check(criterion)
# Verify each returns a CriterionResult (doesn't raise TypeError)
assert hasattr(result, "score")
assert hasattr(result, "criterion_type")
assert result.criterion_type == criterion.type
def test_success_checker_unsupported_type():
"""Test error handling for unsupported criterion type."""
mock_sandbox = Mock()
checker = SuccessChecker(mock_sandbox)
# Create mock criterion with invalid type
bad_criterion = Mock()
bad_criterion.type = "invalid_type_that_does_not_exist"
bad_criterion.description = "Test criterion with unsupported type"
bad_criterion.pass_threshold = 0.9
bad_criterion.is_gating = True # a bare Mock() yields a Mock here, not a bool
# Should return failed result instead of raising
result = checker.check(bad_criterion)
assert result.score == 0.0
assert "Unsupported criterion type" in result.error
assert result.criterion_type == "invalid_type_that_does_not_exist"
def test_success_checker_with_mocked_sandbox():
"""Test that checker can be tested in isolation with mocked sandbox."""
# This demonstrates the key benefit of the refactoring:
# Easy testing with mocked dependencies
mock_sandbox = Mock()
mock_sandbox.file_exists.return_value = True
checker = SuccessChecker(mock_sandbox)
criterion = FileExistsCriterion(path="test.txt", description="Test file")
result = checker.check(criterion)
assert result.score == 1.0
mock_sandbox.file_exists.assert_called_once_with("test.txt")
assert "exists" in result.details
def test_success_checker_mocked_file_contains():
"""Test file contains logic with mocked sandbox."""
mock_sandbox = Mock()
mock_sandbox.file_exists.return_value = True
mock_sandbox.get_file_content.return_value = "Hello World! This is a test."
checker = SuccessChecker(mock_sandbox)
# Test successful match
criterion = FileContainsCriterion(path="test.txt", includes=["Hello", "World"], description="Should match")
result = checker.check(criterion)
assert result.score == 1.0
# Test missing include
criterion = FileContainsCriterion(path="test.txt", includes=["Hello", "Missing"], description="Should fail")
result = checker.check(criterion)
# includes: 1/2=0.5, no excludes so score = includes_score only
assert result.score == 0.5
assert "Missing" in result.details or "1/2" in result.details
def test_success_checker_mocked_command_execution():
"""Test command execution with mocked sandbox."""
mock_sandbox = Mock()
mock_sandbox.run_command.return_value = (0, "Success output", "")
checker = SuccessChecker(mock_sandbox)
criterion = RunCommandCriterion(command="echo test", expected_exit_code=0, description="Test command")
result = checker.check(criterion)
assert result.score == 1.0
mock_sandbox.run_command.assert_called_once_with("echo test", timeout=30)
assert "Exit code: 0" in result.details
def test_handle_criterion_errors_decorator():
"""Test that the @handle_criterion_errors decorator handles exceptions correctly."""
mock_sandbox = Mock()
# Make file_exists raise an exception to test decorator error handling
mock_sandbox.file_exists.side_effect = RuntimeError("Simulated sandbox error")
checker = SuccessChecker(mock_sandbox)
# Test that FileExistsCriterion check catches exception via decorator
criterion = FileExistsCriterion(path="test.txt", description="Test file")
result = checker.check(criterion)
# Verify decorator caught exception and returned failed result
assert result.score == 0.0
assert result.error is not None
assert "Simulated sandbox error" in result.error
assert result.criterion_type == "file_exists"
assert result.description == "Test file"
def test_handle_criterion_errors_decorator_command():
"""Test decorator error handling for command execution criterion."""
mock_sandbox = Mock()
# Make run_command raise a timeout exception
mock_sandbox.run_command.side_effect = TimeoutError("Command timed out after 30s")
checker = SuccessChecker(mock_sandbox)
criterion = RunCommandCriterion(command="long_running_task", expected_exit_code=0, description="Timeout test")
result = checker.check(criterion)
# Verify decorator caught exception and returned failed result
assert result.score == 0.0
assert result.error is not None
assert "Command timed out" in result.error
assert result.criterion_type == "run_command"
def test_handle_criterion_errors_decorator_file_contains():
"""Test decorator error handling for file_contains criterion."""
mock_sandbox = Mock()
mock_sandbox.file_exists.return_value = True
# Make get_file_content raise an exception
mock_sandbox.get_file_content.side_effect = PermissionError("Access denied")
checker = SuccessChecker(mock_sandbox)
criterion = FileContainsCriterion(path="test.txt", includes=["test"], description="Permission error test")
result = checker.check(criterion)
# Verify decorator caught exception and returned failed result
assert result.score == 0.0
assert result.error is not None
assert "Access denied" in result.error
assert result.criterion_type == "file_contains"
def test_success_checker_logs_carry_task_id_in_context(tmp_path):
"""SuccessChecker logs carry task_id when run inside task_log_handler."""
from coder_eval.logging_config import setup_logging, task_log_handler
setup_logging(level="INFO")
mock_sandbox = Mock()
mock_sandbox.file_exists.return_value = True
checker = SuccessChecker(mock_sandbox)
log_file = tmp_path / "task.log"
with task_log_handler(log_file, task_id="my-test-task"):
checker.check(FileExistsCriterion(path="test.txt", description="Test file"))
assert "file_exists" in log_file.read_text()
def test_success_checker_works_without_task_context():
"""SuccessChecker works normally outside any task_log_handler context."""
mock_sandbox = Mock()
mock_sandbox.file_exists.return_value = True
checker = SuccessChecker(mock_sandbox)
result = checker.check(FileExistsCriterion(path="test.txt", description="Test file"))
assert result.score == 1.0