sora/evalstone
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: update evalscope runner, thinking config and webui

Browse Source
This commit is contained in:
sora 2026-07-31 02:36:51 +00:00
parent bde21f6624
commit b1eb4e116a
9 changed files with 281 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bash/case/DP4-flash-int8-thinking-official.sh
View File

@ -1,6 +1,2 @@
python bash/run.py --datasets swe_bench_verified --folder-name DP4-flash-int8-not-thinking
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add0 --thinking --max-tokens-add 0
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add16k --thinking --max-tokens-add 16384
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add32k --thinking --max-tokens-add 32768
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add64k --thinking --max-tokens-add 65536
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add128k --thinking --max-tokens-add 131072
python bash/run.py --datasets live_code_bench --folder-name DP4-flash-int8-thinking --thinking --config /data1/sora/evalscope/config/dpv4-int8_thinking.yaml

65
bash/collect_results.py
View File

@ -206,17 +206,23 @@ def find_archive_predictions(output_dir: Path, benchmark: str, model_name: str):
def find_all_predictions(output_dir: Path, benchmark: str, model_name: str):
"""Find all predictions JSONL files for a benchmark/model.
The durable ``predictions_archive/<benchmark>__<model>.jsonl`` (if
present) is returned **first** so its deduplicated per-sample records
dominate any smaller predictions that a fresh run may have written.
For single-seed runs the durable ``predictions_archive`` is preferred so
breakpoint-resume does not lose completed samples. When multiple seed/run
directories exist (multi-run benchmarks) we aggregate from each run
separately and skip the archive, because the archive only keeps the latest
record per ``index`` and would otherwise collide with one of the runs.
"""
files = list(find_archive_predictions(output_dir, benchmark, model_name))
bench_dir = output_dir / benchmark
if not bench_dir.exists():
return files
for seed_dir in sorted(bench_dir.iterdir()):
if not seed_dir.is_dir():
continue
seed_dirs = []
if bench_dir.exists():
seed_dirs = sorted([p for p in bench_dir.iterdir() if p.is_dir()])
files = []
# Only rely on the archive for single-run / resume scenarios.
if len(seed_dirs) <= 1:
files = list(find_archive_predictions(output_dir, benchmark, model_name))
for seed_dir in seed_dirs:
pred_dir = seed_dir / 'predictions'
if pred_dir.exists():
files.extend(sorted(pred_dir.rglob('*.jsonl')))
@ -283,11 +289,14 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str):
# Aggregate raw prediction perf metrics across all seeds/runs.
# This fixes the breakpoint-resume issue where cumulative stats are reset.
# Predictions are deduplicated by sample `index` so the archive (which
# spans every run) and the latest ``predictions/*.jsonl`` don't double
# count the same sample.
# Deduplicate by (run, sample `index`) so that:
# 1. multiple prediction jsonl files inside the same run directory
# (e.g. `<benchmark>__<model>.jsonl` and `<benchmark>_<subset>.jsonl`)
# do not double-count the same sample;
# 2. each seed/run still contributes its own predictions for multi-run
# benchmarks, so total sample count is sum(runs).
pred_files = find_all_predictions(output_dir, benchmark, model_name)
seen_indexes = set()
seen_keys = set()
latencies = []
ttfts = []
tpots = []
@ -295,30 +304,42 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str):
output_tokens = []
sample_indexes = []
for pf in pred_files:
# ``run_key`` is the seed/run directory name, or the archive filename
# for archive-only scenarios.
run_key = pf.name
parts = pf.parts
if 'predictions_archive' not in parts:
for i, part in enumerate(parts):
if part == 'predictions' and i > 0:
run_key = parts[i - 1]
break
for obj in read_predictions_with_index(pf):
idx = obj['index']
if idx is None or idx in seen_indexes:
# Still keep perf data even when we lack an index, so older
key = (run_key, idx)
if idx is None:
# Keep perf data even when we lack an index, so older
# benchmark files without `index` don't get dropped.
pm = obj['perf_metrics']
elif key in seen_keys:
continue
else:
seen_indexes.add(idx)
seen_keys.add(key)
sample_indexes.append(idx)
pm = obj['perf_metrics']
if pm is None:
continue
if 'latency' in pm:
if pm.get('latency') is not None:
latencies.append(float(pm['latency']))
if 'ttft' in pm:
if pm.get('ttft') is not None:
ttfts.append(float(pm['ttft']))
if 'tpot' in pm:
if pm.get('tpot') is not None:
tpots.append(float(pm['tpot']))
itok = pm.get('input_tokens')
otok = pm.get('output_tokens')
if itok is None and 'usage' in pm:
itok = pm['usage'].get('input_tokens')
otok = pm['usage'].get('output_tokens')
if (itok is None or otok is None) and 'usage' in pm:
itok = pm['usage'].get('input_tokens') if itok is None else itok
otok = pm['usage'].get('output_tokens') if otok is None else otok
if itok is not None:
input_tokens.append(int(itok))
if otok is not None:

108
bash/perf_backup.py
View File

@ -20,7 +20,9 @@ breakpoint-resume issue can be reconstructed:
"""
import json
import os
import shutil
import threading
from datetime import datetime
from pathlib import Path
@ -86,6 +88,20 @@ def backup_perf_stats(output_dir: Path, benchmark: str, model_name: str,
return perf_stats_path
# Per-archive lock to prevent concurrent read-modify-write races when multiple
# seeds of the same benchmark finish at the same time.
_archive_locks: dict = {}
_archive_locks_lock = threading.Lock()
def _get_archive_lock(archive_path: Path) -> threading.Lock:
key = str(archive_path)
with _archive_locks_lock:
if key not in _archive_locks:
_archive_locks[key] = threading.Lock()
return _archive_locks[key]
def archive_predictions(output_dir: Path, benchmark: str, model_name: str,
predictions_dir: Path) -> Path:
"""Append the latest predictions into the durable archive.
@ -95,6 +111,8 @@ def archive_predictions(output_dir: Path, benchmark: str, model_name: str,
the same ``index`` was already archived with a different perf payload,
the newer record wins.
This function is thread-safe for concurrent updates to the same archive.
Returns the archive path.
"""
_, archive_path = get_backup_paths(Path(output_dir), benchmark, model_name)
@ -103,51 +121,55 @@ def archive_predictions(output_dir: Path, benchmark: str, model_name: str,
if not predictions_dir or not predictions_dir.exists():
return archive_path
# Load existing archive (index -> record)
existing = {}
if archive_path.exists():
with archive_path.open('r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
idx = obj.get('index')
if idx is not None:
lock = _get_archive_lock(archive_path)
with lock:
# Load existing archive (index -> record)
existing = {}
if archive_path.exists():
with archive_path.open('r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
idx = obj.get('index')
if idx is not None:
existing[idx] = obj
# Add new predictions
new_count = 0
updated_count = 0
for jsonl in sorted(predictions_dir.rglob('*.jsonl')):
with jsonl.open('r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
idx = obj.get('index')
if idx is None:
continue
if idx in existing:
updated_count += 1
else:
new_count += 1
existing[idx] = obj
# Add new predictions
new_count = 0
updated_count = 0
for jsonl in sorted(predictions_dir.rglob('*.jsonl')):
with jsonl.open('r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
idx = obj.get('index')
if idx is None:
continue
if idx in existing:
updated_count += 1
else:
new_count += 1
existing[idx] = obj
# Write back atomically
tmp_path = archive_path.with_suffix('.jsonl.tmp')
with tmp_path.open('w', encoding='utf-8') as f:
for idx in sorted(existing.keys()):
f.write(json.dumps(existing[idx], ensure_ascii=False))
f.write('\n')
shutil.move(str(tmp_path), str(archive_path))
# Write back atomically using a unique temp file per thread to avoid
# collisions when multiple runs update the same archive.
tmp_path = archive_path.with_suffix(
f'.jsonl.tmp.{os.getpid()}.{threading.get_ident()}')
with tmp_path.open('w', encoding='utf-8') as f:
for idx in sorted(existing.keys()):
f.write(json.dumps(existing[idx], ensure_ascii=False))
f.write('\n')
shutil.move(str(tmp_path), str(archive_path))
return archive_path

119
bash/run.py
View File

@ -31,6 +31,7 @@ Examples:
import argparse
import json
import sys
import threading
import time
from copy import deepcopy
from pathlib import Path
@ -243,6 +244,16 @@ def build_parser():
help='Random seed (default: %(default)s)')
parser.add_argument('--batch-size', type=int, default=DEFAULT_BATCH_SIZE,
help='Evaluation batch size (default: %(default)s)')
parser.add_argument('--parallel-runs', type=int, default=1,
help='Number of multi-run seeds to execute in parallel '
'(default: %(default)s). Each run still uses --batch-size '
'concurrent requests, so total API concurrency is '
'parallel-runs * batch-size.')
parser.add_argument('--parallel-benchmarks', type=int, default=1,
help='Number of different benchmarks to execute in parallel '
'(default: %(default)s). When >1, each benchmark runs in its '
'own thread; completed runs immediately free a slot for the '
'next benchmark.')
# Decoding / thinking
parser.add_argument('--thinking', action='store_true', default=None,
@ -537,7 +548,8 @@ def write_summary(output_dir: str, model_name: str, folder_name: str,
def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str,
model_name: str, folder_name: str, benchmark_names: list):
model_name: str, folder_name: str, benchmark_names: list,
summary_lock: threading.Lock = None):
"""Run a benchmark task and optionally refresh the summary table.
``benchmark_names`` is the running list of canonical benchmark names that
@ -549,6 +561,9 @@ def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str,
``perf_metrics.summary`` and the per-sample predictions into the durable
backup files maintained by ``perf_backup.py`` so checkpoint restarts can
recover them.
If ``summary_lock`` is provided, the summary write is protected so multiple
seeds/runs can safely refresh the table as each one finishes.
"""
dataset_name = task_cfg.datasets[0]
work_dir = Path(task_cfg.work_dir)
@ -564,7 +579,11 @@ def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str,
backup_after_run(output_dir, dataset_name, model_name, work_dir)
if write_summary_flag:
write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names)
if summary_lock is not None:
with summary_lock:
write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names)
else:
write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names)
def backup_after_run(output_dir: str, benchmark: str, model_name: str,
@ -710,7 +729,8 @@ def main():
f'max_tokens={DEFAULT_GENERATION_CONFIG["max_tokens"]})')
return {'generation_config': deepcopy(DEFAULT_GENERATION_CONFIG)}
def run_one(dataset_name, run_idx=0, benchmark_names=None):
def run_one(dataset_name, run_idx=0, benchmark_names=None, write_summary_flag=True,
summary_lock=None):
ds_cfg = get_dataset_config(dataset_name)
task_cfg = build_task_config(
dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
@ -720,8 +740,9 @@ def main():
max_tokens_add=args.max_tokens_add,
)
try:
run_and_summarize(task_cfg, args.write_summary, str(model_output_dir), args.model,
folder_name=folder_name, benchmark_names=benchmark_names)
run_and_summarize(task_cfg, write_summary_flag, str(model_output_dir), args.model,
folder_name=folder_name, benchmark_names=benchmark_names,
summary_lock=summary_lock)
except Exception as e:
print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}')
@ -729,31 +750,89 @@ def main():
# the summary table only includes them, not stale results from earlier runs.
benchmark_names = []
# Build a list of benchmark execution units. Each unit runs one complete
# benchmark (all its multi-runs). This lets us run benchmarks either
# serially or in parallel while keeping per-benchmark run parallelism
# (``--parallel-runs``) intact.
benchmark_units = []
for dataset_name in multi_run:
benchmark_names.append(dataset_name)
num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1)
for run_idx in range(num_runs):
print(f"\n{'='*60}")
print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})')
print(f"{'='*60}")
run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names)
benchmark_units.append((dataset_name, 'multi'))
for dataset_name in single_run:
benchmark_names.append(dataset_name)
print(f"\n{'='*60}")
print(f'Running: {dataset_name} (seed={args.seed})')
print(f"{'='*60}")
run_one(dataset_name, benchmark_names=benchmark_names)
benchmark_units.append((dataset_name, 'single'))
for dataset_name in agent:
benchmark_names.append(dataset_name)
benchmark_units.append((dataset_name, 'agent'))
def run_benchmark_unit(dataset_name: str, kind: str, summary_lock=None):
"""Run one benchmark (all seeds/runs) and return its name."""
if kind == 'multi':
num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1)
parallel_runs = max(1, args.parallel_runs)
if parallel_runs <= 1:
for run_idx in range(num_runs):
print(f"\n{'='*60}")
print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})')
print(f"{'='*60}")
run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names,
write_summary_flag=True, summary_lock=summary_lock)
else:
print(f"\n{'='*60}")
print(f'Running: {dataset_name} ({num_runs} runs, {parallel_runs} in parallel, '
f'seed={args.seed}, per-run batch_size={args.batch_size})')
print(f"{'='*60}")
from concurrent.futures import ThreadPoolExecutor, as_completed
def _run_single(run_idx):
print(f' -> start run {run_idx + 1}/{num_runs}')
run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names,
write_summary_flag=True, summary_lock=summary_lock)
print(f' -> finish run {run_idx + 1}/{num_runs}')
with ThreadPoolExecutor(max_workers=parallel_runs) as executor:
futures = [executor.submit(_run_single, run_idx) for run_idx in range(num_runs)]
for future in as_completed(futures):
try:
future.result()
except Exception as e:
print(f'ERROR in {dataset_name} parallel run: {e}')
else:
print(f"\n{'='*60}")
print(f'Running: {dataset_name} (seed={args.seed})')
print(f"{'='*60}")
run_one(dataset_name, benchmark_names=benchmark_names, write_summary_flag=True,
summary_lock=summary_lock)
return dataset_name
# Lock to protect concurrent writes to the summary CSV/Excel when multiple
# seeds/runs or benchmarks finish around the same time.
summary_lock = threading.Lock()
parallel_benchmarks = max(1, args.parallel_benchmarks)
if parallel_benchmarks <= 1:
for dataset_name, kind in benchmark_units:
run_benchmark_unit(dataset_name, kind, summary_lock=summary_lock)
else:
print(f"\n{'='*60}")
print(f'Running: {dataset_name} (seed={args.seed})')
print(f'Running benchmarks in parallel: {len(benchmark_units)} units, '
f'{parallel_benchmarks} at a time')
print(f"{'='*60}")
run_one(dataset_name, benchmark_names=benchmark_names)
from concurrent.futures import ThreadPoolExecutor, as_completed
with ThreadPoolExecutor(max_workers=parallel_benchmarks) as executor:
futures = [executor.submit(run_benchmark_unit, dataset_name, kind, summary_lock)
for dataset_name, kind in benchmark_units]
for future in as_completed(futures):
try:
dataset_name = future.result()
print(f' -> benchmark finished: {dataset_name}')
except Exception as e:
print(f'ERROR in parallel benchmark: {e}')
if args.write_summary:
write_summary(str(model_output_dir), args.model, folder_name, benchmark_names=benchmark_names)
with summary_lock:
write_summary(str(model_output_dir), args.model, folder_name, benchmark_names=benchmark_names)
print('\nAll benchmarks done!')

31
config/dpv4-int8_thinking.yaml
View File

@ -4,54 +4,63 @@ gpqa_diamond:
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
hle:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
aime24:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
aime25:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 65536
max_completion_tokens: 64000
mmlu_pro:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
simple_qa:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
arc:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
bbh:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
live_code_bench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 65536
max_completion_tokens: 64000
dataset_args:
subset_list:
- release_v6
@ -61,96 +70,112 @@ aime26:
top_p: 1.0
stream: true
max_tokens: 65536
max_completion_tokens: 64000
hmmt26:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
imo_answerbench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
super_gpqa:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
drop:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
hellaswag:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
mmlu:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
openai_mrcr:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
bigcodebench:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
humaneval:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
gsm8k:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
competition_math:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
cmmlu:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
trivia_qa:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
winogrande:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
longbench_v2:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
max_completion_tokens: 64000
dataset_args:
subset_list:
- short
@ -162,6 +187,7 @@ tau2_bench:
top_p: 1.0
stream: true
max_tokens: 16384
max_completion_tokens: 64000
dataset_args:
extra_params:
user_model: deepseek-v4-pro
@ -170,6 +196,7 @@ tau2_bench:
generation_config:
temperature: 0.0
max_tokens: 4096
max_completion_tokens: 64000
agent_config:
mode: native
strategy: react
@ -180,12 +207,14 @@ general_fc:
top_p: 1.0
stream: true
max_tokens: 4096
max_completion_tokens: 64000
bfcl_v3:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 4096
max_completion_tokens: 64000
parallel_tool_calls: true
dataset_args:
extra_params:
@ -197,9 +226,11 @@ swe_bench_verified:
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000
swe_bench_pro:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
max_completion_tokens: 64000

3
evalscope/evalscope/api/model/generate_config.py
View File

@ -51,6 +51,9 @@ class GenerateConfig(BaseModel):
max_tokens: Optional[int] = Field(default=None)
"""The maximum number of tokens that can be generated in the completion (default is model specific)."""
max_completion_tokens: Optional[int] = Field(default=None)
"""An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens. Used by OpenAI reasoning models and some proxies."""
top_p: Optional[float] = Field(default=None)
"""An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass."""

3
evalscope/evalscope/api/model/model.py
View File

@ -306,6 +306,9 @@ def get_model_with_task_config(task_config: 'TaskConfig') -> Model:
eval_type = task_config.eval_type
base_url = task_config.api_url
api_key = task_config.api_key
# TaskConfig defaults api_key to 'EMPTY'; treat it as unset so env vars can be used.
if api_key == 'EMPTY':
api_key = None
config = task_config.generation_config
model_args = task_config.model_args or {}

52
evalscope/evalscope/models/utils/openai.py
View File

@ -256,6 +256,8 @@ def openai_completion_params(model: str, config: GenerateConfig, tools: bool) ->
params['timeout'] = config.timeout
if config.max_tokens is not None:
params['max_tokens'] = config.max_tokens
if config.max_completion_tokens is not None:
params['max_completion_tokens'] = config.max_completion_tokens
if config.frequency_penalty is not None:
params['frequency_penalty'] = config.frequency_penalty
if config.stop_seqs is not None:
@ -822,6 +824,9 @@ def collect_stream_response(
continue
collected_chunks.append(chunk)
for choice in chunk.choices:
# Some proxies (e.g. Vectron) omit choice.index; default to 0.
choice_index = choice.index if choice.index is not None else 0
# Detect first meaningful content chunk for TTFT
has_content = ((choice.delta.content is not None and choice.delta.content != '') or (
hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None
@ -832,11 +837,11 @@ def collect_stream_response(
# Handle reasoning content
if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None:
collected_reasoning[choice.index].append(choice.delta.reasoning_content)
collected_reasoning[choice_index].append(choice.delta.reasoning_content)
# Handle regular content
if choice.delta.content is not None:
collected_messages[choice.index].append(choice.delta.content)
collected_messages[choice_index].append(choice.delta.content)
# Handle tool calls
if hasattr(choice.delta, 'tool_calls') and choice.delta.tool_calls:
@ -844,8 +849,8 @@ def collect_stream_response(
tool_id = tool_call.index
# Initialize tool call if not present
if tool_id not in collected_tool_calls[choice.index]:
collected_tool_calls[choice.index][tool_id] = {
if tool_id not in collected_tool_calls[choice_index]:
collected_tool_calls[choice_index][tool_id] = {
'id': tool_call.id if hasattr(tool_call, 'id') and tool_call.id else None,
'type': tool_call.type if hasattr(tool_call, 'type') and tool_call.type else None,
'function': {
@ -857,15 +862,15 @@ def collect_stream_response(
# Update tool call with new chunks
if hasattr(tool_call, 'function'):
if hasattr(tool_call.function, 'name') and tool_call.function.name:
collected_tool_calls[choice.index][tool_id]['function']['name'] = tool_call.function.name
collected_tool_calls[choice_index][tool_id]['function']['name'] = tool_call.function.name
if hasattr(tool_call.function, 'arguments') and tool_call.function.arguments:
collected_tool_calls[choice.index
collected_tool_calls[choice_index
][tool_id]['function']['arguments'] += tool_call.function.arguments
# Update ID if it was received later
if hasattr(tool_call, 'id') and tool_call.id:
collected_tool_calls[choice.index][tool_id]['id'] = tool_call.id
collected_tool_calls[choice_index][tool_id]['id'] = tool_call.id
# Get all unique choice indices from all collections
all_indices = set(collected_messages.keys()) | set(collected_reasoning.keys()) | set(collected_tool_calls.keys())
@ -885,9 +890,11 @@ def collect_stream_response(
# use the finish_reason from the last chunk that generated this choice
finish_reason = None
for chunk in reversed(collected_chunks):
if chunk.choices and chunk.choices[0].index == index:
finish_reason = chunk.choices[0].finish_reason
break
if chunk.choices:
chunk_index = chunk.choices[0].index if chunk.choices[0].index is not None else 0
if chunk_index == index:
finish_reason = chunk.choices[0].finish_reason
break
message_kwargs = {'role': 'assistant', 'content': full_reply_content}
@ -950,6 +957,9 @@ async def async_collect_stream_response(
continue
collected_chunks.append(chunk)
for choice in chunk.choices:
# Some proxies (e.g. Vectron) omit choice.index; default to 0.
choice_index = choice.index if choice.index is not None else 0
# Detect first meaningful content chunk for TTFT
has_content = ((choice.delta.content is not None and choice.delta.content != '') or (
hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None
@ -960,11 +970,11 @@ async def async_collect_stream_response(
# Handle reasoning content
if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None:
collected_reasoning[choice.index].append(choice.delta.reasoning_content)
collected_reasoning[choice_index].append(choice.delta.reasoning_content)
# Handle regular content
if choice.delta.content is not None:
collected_messages[choice.index].append(choice.delta.content)
collected_messages[choice_index].append(choice.delta.content)
# Handle tool calls
if hasattr(choice.delta, 'tool_calls') and choice.delta.tool_calls:
@ -972,8 +982,8 @@ async def async_collect_stream_response(
tool_id = tool_call.index
# Initialize tool call if not present
if tool_id not in collected_tool_calls[choice.index]:
collected_tool_calls[choice.index][tool_id] = {
if tool_id not in collected_tool_calls[choice_index]:
collected_tool_calls[choice_index][tool_id] = {
'id': tool_call.id if hasattr(tool_call, 'id') and tool_call.id else None,
'type': tool_call.type if hasattr(tool_call, 'type') and tool_call.type else None,
'function': {
@ -985,15 +995,15 @@ async def async_collect_stream_response(
# Update tool call with new chunks
if hasattr(tool_call, 'function'):
if hasattr(tool_call.function, 'name') and tool_call.function.name:
collected_tool_calls[choice.index][tool_id]['function']['name'] = tool_call.function.name
collected_tool_calls[choice_index][tool_id]['function']['name'] = tool_call.function.name
if hasattr(tool_call.function, 'arguments') and tool_call.function.arguments:
collected_tool_calls[choice.index
collected_tool_calls[choice_index
][tool_id]['function']['arguments'] += tool_call.function.arguments
# Update ID if it was received later
if hasattr(tool_call, 'id') and tool_call.id:
collected_tool_calls[choice.index][tool_id]['id'] = tool_call.id
collected_tool_calls[choice_index][tool_id]['id'] = tool_call.id
# Get all unique choice indices from all collections
all_indices = set(collected_messages.keys()) | set(collected_reasoning.keys()) | set(collected_tool_calls.keys())
@ -1013,9 +1023,11 @@ async def async_collect_stream_response(
# use the finish_reason from the last chunk that generated this choice
finish_reason = None
for chunk in reversed(collected_chunks):
if chunk.choices and chunk.choices[0].index == index:
finish_reason = chunk.choices[0].finish_reason
break
if chunk.choices:
chunk_index = chunk.choices[0].index if chunk.choices[0].index is not None else 0
if chunk_index == index:
finish_reason = chunk.choices[0].finish_reason
break
message_kwargs = {'role': 'assistant', 'content': full_reply_content}

2
webui/server.py
View File

@ -408,6 +408,8 @@ async def start_job(req: LaunchRequest) -> JobRecord:
env['PYTHONUNBUFFERED'] = '1'
if req.api_key and req.api_key != 'EMPTY':
env['OPENAI_API_KEY'] = req.api_key
# EvalScope's openai_api backend reads EVALSCOPE_API_KEY.
env['EVALSCOPE_API_KEY'] = req.api_key
try:
proc = await asyncio.create_subprocess_exec(