95 lines
3.6 KiB
Python
95 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Fill simple_qa (fresh re-run) + bfcl_v3 (official OVERALL) into the Excel.
|
|
|
|
simple_qa: old cells had D=0.0005 (cache-resume artifact) and empty perf.
|
|
Overwrite C/D/F-Q from the NEW report (real perf + ~1.9h duration).
|
|
bfcl_v3 : top-level score 0.6643 is evalscope's macro-average (non-standard).
|
|
Official BFCL score = OVERALL subset = 0.569. Update C only.
|
|
"""
|
|
import json
|
|
import glob
|
|
import openpyxl
|
|
|
|
XLSX = '/data1/sora/P800模型能力评测结果_统一格式_filled.xlsx'
|
|
SHEETS = ['2.0-FULL', '2.0-Lite']
|
|
|
|
# --- read reports ---
|
|
sq = json.load(open('/data1/sora/evalscope/output/simple_qa/seed_42/reports/DeepSeek-V4-Flash-Int8/simple_qa.json'))
|
|
bc = json.load(open(glob.glob('/data1/sora/evalscope/output/bfcl_v3/seed_42.bak/reports/DeepSeek-V4-Flash-Int8/bfcl_v3.json')[0]))
|
|
|
|
|
|
def perf(d):
|
|
pm = d.get('perf_metrics') or {}
|
|
s = pm.get('summary', {}) or {}
|
|
u = s.get('usage', {}) or {}
|
|
tf = s.get('ttft') or {}
|
|
tp = s.get('tpot') or {}
|
|
lat = (s.get('latency') or {}).get('mean')
|
|
return {
|
|
'score': d.get('score'),
|
|
'duration_h': (d.get('duration_sec') or 0) / 3600,
|
|
'latency': lat,
|
|
'tps': (s.get('throughput') or {}).get('avg_output_tps'),
|
|
'qps': (1.0 / lat) if lat else None,
|
|
'in_tok': (u.get('input_tokens') or {}).get('mean'),
|
|
'out_tok': (u.get('output_tokens') or {}).get('mean'),
|
|
'ttc': u.get('total_tokens_count'),
|
|
'ttft_m': tf.get('mean'),
|
|
'ttft90': tf.get('90%') or tf.get('p90'),
|
|
'ttft99': tf.get('99%') or tf.get('p99'),
|
|
'tpot_m': tp.get('mean'),
|
|
'tpot90': tp.get('90%') or tp.get('p90'),
|
|
'tpot99': tp.get('99%') or tp.get('p99'),
|
|
}
|
|
|
|
|
|
def bfcl_overall(d):
|
|
for metric in d.get('metrics', []):
|
|
for cat in metric.get('categories', []):
|
|
for sub in cat.get('subsets', []):
|
|
nm = sub.get('name')
|
|
if nm == 'OVERALL' or (isinstance(nm, list) and 'OVERALL' in nm):
|
|
return sub.get('score')
|
|
return None
|
|
|
|
|
|
P = perf(sq)
|
|
BC_OVERALL = bfcl_overall(bc)
|
|
print('simple_qa:', {k: (round(v, 4) if isinstance(v, float) else v) for k, v in P.items()})
|
|
print('bfcl_v3 OVERALL:', BC_OVERALL)
|
|
|
|
# col -> simple_qa perf key
|
|
SQ = {'C': P['score'], 'D': round(P['duration_h'], 4), 'F': P['latency'], 'G': P['tps'],
|
|
'H': P['qps'], 'I': P['in_tok'], 'J': P['out_tok'], 'K': P['ttc'],
|
|
'L': P['ttft_m'], 'M': P['ttft90'], 'N': P['ttft99'],
|
|
'O': P['tpot_m'], 'P': P['tpot90'], 'Q': P['tpot99']}
|
|
|
|
wb = openpyxl.load_workbook(XLSX)
|
|
for sn in SHEETS:
|
|
ws = wb[sn]
|
|
for r in range(2, 30):
|
|
b = ws.cell(r, 2).value
|
|
if b == 'simple_qa':
|
|
for col, v in SQ.items():
|
|
if v is not None:
|
|
ws[f'{col}{r}'] = int(round(v)) if col == 'K' else round(v, 4) if col in ('C', 'D') else v
|
|
print(f'{sn} simple_qa row{r}: C={ws[f"C{r}"].value} D={ws[f"D{r}"].value} F={ws[f"F{r}"].value} K={ws[f"K{r}"].value}')
|
|
elif b == 'bfcl_v3' and BC_OVERALL is not None:
|
|
ws.cell(r, 3).value = round(BC_OVERALL, 4)
|
|
print(f'{sn} bfcl_v3 row{r}: C={ws.cell(r,3).value}')
|
|
# recompute total D
|
|
tr = next((rr for rr in range(2, ws.max_row + 1) if ws.cell(rr, 2).value == '总计'), None)
|
|
if tr:
|
|
tot = 0.0
|
|
for rr in range(2, tr):
|
|
v = ws.cell(rr, 4).value
|
|
try:
|
|
tot += float(v)
|
|
except (TypeError, ValueError):
|
|
pass
|
|
ws.cell(tr, 4).value = round(tot, 4)
|
|
print(f'{sn} 总计 D = {round(tot, 4)}h')
|
|
|
|
wb.save(XLSX)
|
|
print('saved ->', XLSX)
|