119 lines
4.4 KiB
Python
119 lines
4.4 KiB
Python
"""Model layer tests: spec parsing, adapters, async runner end-to-end.
|
|
|
|
Run: .venv/bin/python tests/test_model.py
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import sys
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from evalharness.data.sample import Sample, ToolInfo # noqa: E402
|
|
from evalharness.model import MockAdapter, OpenAICompatible, resolve_adapter, run_eval # noqa: E402
|
|
from evalharness.model.adapter import parse_model_spec # noqa: E402
|
|
from evalharness.model.runner import _make_adapter # noqa: E402
|
|
|
|
|
|
def test_parse_spec():
|
|
got = parse_model_spec('openai/http://gpu03:8000/v1?qwen3-8b')
|
|
assert got == {'adapter': 'openai', 'api_base': 'http://gpu03:8000/v1', 'model': 'qwen3-8b'}
|
|
assert parse_model_spec('mock') == {'adapter': 'mock', 'api_base': '', 'model': ''}
|
|
|
|
|
|
def test_make_adapter_mock_modes():
|
|
a = _make_adapter('mock:boxed')
|
|
assert isinstance(a, MockAdapter) and a.extra['mode'] == 'boxed'
|
|
b = _make_adapter('mock')
|
|
assert isinstance(b, MockAdapter) # mode defaults to echo at generate time
|
|
c = _make_adapter('openai/http://127.0.0.1:9/v1?m')
|
|
assert isinstance(c, OpenAICompatible) and c.api_base == 'http://127.0.0.1:9/v1' and c.model == 'm'
|
|
|
|
|
|
def test_mock_tool_mode():
|
|
out = asyncio.run(MockAdapter().generate([Sample(input='x').input
|
|
and __import__('evalharness.data.sample', fromlist=['ChatMessage']).ChatMessage(role='user', content='x')]))
|
|
assert out.text == 'x'
|
|
|
|
|
|
def _fake_server():
|
|
class Fake(BaseHTTPRequestHandler):
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
|
msg = body['messages'][-1]['content']
|
|
tool_calls = []
|
|
if body.get('tools'):
|
|
tool_calls = [{'id': 'c1', 'function': {'name': 't1', 'arguments': '{"x": 1}'}}]
|
|
resp = {'choices': [{'message': {'content': f'echo:{msg}', 'tool_calls': tool_calls},
|
|
'finish_reason': 'tool_calls' if tool_calls else 'stop'}],
|
|
'usage': {'prompt_tokens': 3, 'completion_tokens': 5, 'total_tokens': 8},
|
|
'model': body['model']}
|
|
data = json.dumps(resp).encode()
|
|
self.send_response(200)
|
|
self.send_header('Content-Type', 'application/json')
|
|
self.send_header('Content-Length', str(len(data)))
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
|
|
def log_message(self, *a):
|
|
pass
|
|
|
|
srv = HTTPServer(('127.0.0.1', 0), Fake)
|
|
threading.Thread(target=srv.serve_forever, daemon=True).start()
|
|
return srv
|
|
|
|
|
|
def test_openai_adapter_roundtrip():
|
|
srv = _fake_server()
|
|
port = srv.server_address[1]
|
|
try:
|
|
rep = asyncio.run(run_eval([Sample(input='hello', target='echo:hello')],
|
|
f'openai/http://127.0.0.1:{port}/v1?fake-m',
|
|
progress=False))
|
|
r = rep.samples[0]
|
|
assert 'echo:hello' in r.raw_prediction
|
|
assert r.usage['total_tokens'] == 8
|
|
finally:
|
|
srv.shutdown()
|
|
|
|
|
|
def test_tools_pass_through():
|
|
srv = _fake_server()
|
|
port = srv.server_address[1]
|
|
try:
|
|
s = Sample(input='call it', tools=[ToolInfo(name='f1', parameters={'type': 'object'})])
|
|
rep = asyncio.run(run_eval([s], f'openai/http://127.0.0.1:{port}/v1?fake-m',
|
|
progress=False))
|
|
assert '"t1"' in rep.samples[0].raw_prediction # tool call serialized into prediction
|
|
finally:
|
|
srv.shutdown()
|
|
|
|
|
|
def test_oracle_mock_pipeline():
|
|
from evalharness import get_dataset
|
|
|
|
ds = get_dataset('gsm8k')
|
|
rep = asyncio.run(run_eval(ds, 'mock:boxed', limit=50, concurrency=8, progress=False))
|
|
assert rep.metrics['acc'] == 1.0
|
|
assert rep.dataset == 'gsm8k'
|
|
assert rep.metric_groups['run_info']['gen_total_tokens'] == 100
|
|
|
|
|
|
if __name__ == '__main__':
|
|
fails = 0
|
|
for name, fn in sorted({k: v for k, v in globals().items()
|
|
if k.startswith('test_') and callable(v)}.items()):
|
|
try:
|
|
fn()
|
|
print(f'PASS {name}')
|
|
except AssertionError as e:
|
|
fails += 1
|
|
print(f'FAIL {name}: {e}')
|
|
except Exception as e:
|
|
fails += 1
|
|
print(f'ERROR {name}: {type(e).__name__}: {e}')
|
|
sys.exit(1 if fails else 0)
|