Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

JSON-tolerant report/checkpoint saves (default=str)

Browse Source 
A finished 12-minute humaneval bench died at the save line: some scorer
detail carried a non-JSON object (Ellipsis) and report.save's json.dumps
had no default. Every dumps on the save path now stringifies exotic
objects instead of killing the run; verified by replaying the exact
crashed checkpoint end-to-end (pass 82.3%, 15s, no crash).

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-14 07:15:50 +00:00
parent 652c13db36
commit 77d2c4569a
2 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
evalharness/eval/checkpoint.py
View File

@ -70,7 +70,7 @@ class CheckpointStore:
worst case loses the last in-flight sample on crash).""" worst case loses the last in-flight sample on crash)."""
rec = {'key': key, 'ts': time.time(), 'pred': pred} rec = {'key': key, 'ts': time.time(), 'pred': pred}
with open(self.path, 'a', encoding='utf-8') as f: with open(self.path, 'a', encoding='utf-8') as f:
f.write(json.dumps(rec, ensure_ascii=False) + '\n') f.write(json.dumps(rec, ensure_ascii=False, default=str) + '\n')
self._entries[key] = pred self._entries[key] = pred
def scores(self) -> Dict[str, Dict[str, Any]]: def scores(self) -> Dict[str, Dict[str, Any]]:

11
evalharness/eval/record.py
View File

@ -68,20 +68,25 @@ class EvalReport(BaseModel):
def save(self, path) -> None: def save(self, path) -> None:
import json import json
# default=str everywhere: score_details/samples carry arbitrary
# scorer output, and ONE exotic object (an Ellipsis sneaked in via
# a scorer's detail dict) must not kill a finished benchmark at
# the save line
if str(path).endswith('.jsonl'): if str(path).endswith('.jsonl'):
# streaming format: first line = report header, then one # streaming format: first line = report header, then one
# sample per line (grep/split/tail friendly) # sample per line (grep/split/tail friendly)
head = self.model_dump(exclude={'samples'}) head = self.model_dump(exclude={'samples'})
head['type'] = 'report' head['type'] = 'report'
with open(path, 'w', encoding='utf-8') as f: with open(path, 'w', encoding='utf-8') as f:
f.write(json.dumps(head, ensure_ascii=False) + '\n') f.write(json.dumps(head, ensure_ascii=False, default=str) + '\n')
for smp in self.samples: for smp in self.samples:
row = smp if isinstance(smp, dict) else smp.model_dump() row = smp if isinstance(smp, dict) else smp.model_dump()
row['type'] = 'sample' row['type'] = 'sample'
f.write(json.dumps(row, ensure_ascii=False) + '\n') f.write(json.dumps(row, ensure_ascii=False, default=str) + '\n')
return return
with open(path, 'w', encoding='utf-8') as f: with open(path, 'w', encoding='utf-8') as f:
json.dump(self.model_dump(), f, ensure_ascii=False, indent=2) json.dump(self.model_dump(), f, ensure_ascii=False, indent=2,
default=str)
@classmethod @classmethod
def load(cls, path) -> 'EvalReport': def load(cls, path) -> 'EvalReport':