From b1eb4e116afd29f897ca6747149c4fbc9d177691 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Fri, 31 Jul 2026 02:36:51 +0000 Subject: [PATCH] chore: update evalscope runner, thinking config and webui --- bash/case/DP4-flash-int8-thinking-official.sh | 8 +- bash/collect_results.py | 65 ++++++---- bash/perf_backup.py | 108 +++++++++------- bash/run.py | 119 +++++++++++++++--- config/dpv4-int8_thinking.yaml | 33 ++++- .../evalscope/api/model/generate_config.py | 3 + evalscope/evalscope/api/model/model.py | 3 + evalscope/evalscope/models/utils/openai.py | 52 +++++--- webui/server.py | 2 + 9 files changed, 281 insertions(+), 112 deletions(-) diff --git a/bash/case/DP4-flash-int8-thinking-official.sh b/bash/case/DP4-flash-int8-thinking-official.sh index 0e5d7ac..07f17e4 100644 --- a/bash/case/DP4-flash-int8-thinking-official.sh +++ b/bash/case/DP4-flash-int8-thinking-official.sh @@ -1,6 +1,2 @@ -python bash/run.py --datasets swe_bench_verified --folder-name DP4-flash-int8-not-thinking -python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add0 --thinking --max-tokens-add 0 -python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add16k --thinking --max-tokens-add 16384 -python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add32k --thinking --max-tokens-add 32768 -python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add64k --thinking --max-tokens-add 65536 -python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add128k --thinking --max-tokens-add 131072 + +python bash/run.py --datasets live_code_bench --folder-name DP4-flash-int8-thinking --thinking --config /data1/sora/evalscope/config/dpv4-int8_thinking.yaml diff --git a/bash/collect_results.py b/bash/collect_results.py index 487215b..56af21f 100644 --- a/bash/collect_results.py +++ b/bash/collect_results.py @@ -206,17 +206,23 @@ def find_archive_predictions(output_dir: Path, benchmark: str, model_name: str): def find_all_predictions(output_dir: Path, benchmark: str, model_name: str): """Find all predictions JSONL files for a benchmark/model. - The durable ``predictions_archive/__.jsonl`` (if - present) is returned **first** so its deduplicated per-sample records - dominate any smaller predictions that a fresh run may have written. + For single-seed runs the durable ``predictions_archive`` is preferred so + breakpoint-resume does not lose completed samples. When multiple seed/run + directories exist (multi-run benchmarks) we aggregate from each run + separately and skip the archive, because the archive only keeps the latest + record per ``index`` and would otherwise collide with one of the runs. """ - files = list(find_archive_predictions(output_dir, benchmark, model_name)) bench_dir = output_dir / benchmark - if not bench_dir.exists(): - return files - for seed_dir in sorted(bench_dir.iterdir()): - if not seed_dir.is_dir(): - continue + seed_dirs = [] + if bench_dir.exists(): + seed_dirs = sorted([p for p in bench_dir.iterdir() if p.is_dir()]) + + files = [] + # Only rely on the archive for single-run / resume scenarios. + if len(seed_dirs) <= 1: + files = list(find_archive_predictions(output_dir, benchmark, model_name)) + + for seed_dir in seed_dirs: pred_dir = seed_dir / 'predictions' if pred_dir.exists(): files.extend(sorted(pred_dir.rglob('*.jsonl'))) @@ -283,11 +289,14 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str): # Aggregate raw prediction perf metrics across all seeds/runs. # This fixes the breakpoint-resume issue where cumulative stats are reset. - # Predictions are deduplicated by sample `index` so the archive (which - # spans every run) and the latest ``predictions/*.jsonl`` don't double - # count the same sample. + # Deduplicate by (run, sample `index`) so that: + # 1. multiple prediction jsonl files inside the same run directory + # (e.g. `__.jsonl` and `_.jsonl`) + # do not double-count the same sample; + # 2. each seed/run still contributes its own predictions for multi-run + # benchmarks, so total sample count is sum(runs). pred_files = find_all_predictions(output_dir, benchmark, model_name) - seen_indexes = set() + seen_keys = set() latencies = [] ttfts = [] tpots = [] @@ -295,30 +304,42 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str): output_tokens = [] sample_indexes = [] for pf in pred_files: + # ``run_key`` is the seed/run directory name, or the archive filename + # for archive-only scenarios. + run_key = pf.name + parts = pf.parts + if 'predictions_archive' not in parts: + for i, part in enumerate(parts): + if part == 'predictions' and i > 0: + run_key = parts[i - 1] + break for obj in read_predictions_with_index(pf): idx = obj['index'] - if idx is None or idx in seen_indexes: - # Still keep perf data even when we lack an index, so older + key = (run_key, idx) + if idx is None: + # Keep perf data even when we lack an index, so older # benchmark files without `index` don't get dropped. pm = obj['perf_metrics'] + elif key in seen_keys: + continue else: - seen_indexes.add(idx) + seen_keys.add(key) sample_indexes.append(idx) pm = obj['perf_metrics'] if pm is None: continue - if 'latency' in pm: + if pm.get('latency') is not None: latencies.append(float(pm['latency'])) - if 'ttft' in pm: + if pm.get('ttft') is not None: ttfts.append(float(pm['ttft'])) - if 'tpot' in pm: + if pm.get('tpot') is not None: tpots.append(float(pm['tpot'])) itok = pm.get('input_tokens') otok = pm.get('output_tokens') - if itok is None and 'usage' in pm: - itok = pm['usage'].get('input_tokens') - otok = pm['usage'].get('output_tokens') + if (itok is None or otok is None) and 'usage' in pm: + itok = pm['usage'].get('input_tokens') if itok is None else itok + otok = pm['usage'].get('output_tokens') if otok is None else otok if itok is not None: input_tokens.append(int(itok)) if otok is not None: diff --git a/bash/perf_backup.py b/bash/perf_backup.py index c05b31f..e62355b 100644 --- a/bash/perf_backup.py +++ b/bash/perf_backup.py @@ -20,7 +20,9 @@ breakpoint-resume issue can be reconstructed: """ import json +import os import shutil +import threading from datetime import datetime from pathlib import Path @@ -86,6 +88,20 @@ def backup_perf_stats(output_dir: Path, benchmark: str, model_name: str, return perf_stats_path +# Per-archive lock to prevent concurrent read-modify-write races when multiple +# seeds of the same benchmark finish at the same time. +_archive_locks: dict = {} +_archive_locks_lock = threading.Lock() + + +def _get_archive_lock(archive_path: Path) -> threading.Lock: + key = str(archive_path) + with _archive_locks_lock: + if key not in _archive_locks: + _archive_locks[key] = threading.Lock() + return _archive_locks[key] + + def archive_predictions(output_dir: Path, benchmark: str, model_name: str, predictions_dir: Path) -> Path: """Append the latest predictions into the durable archive. @@ -95,6 +111,8 @@ def archive_predictions(output_dir: Path, benchmark: str, model_name: str, the same ``index`` was already archived with a different perf payload, the newer record wins. + This function is thread-safe for concurrent updates to the same archive. + Returns the archive path. """ _, archive_path = get_backup_paths(Path(output_dir), benchmark, model_name) @@ -103,51 +121,55 @@ def archive_predictions(output_dir: Path, benchmark: str, model_name: str, if not predictions_dir or not predictions_dir.exists(): return archive_path - # Load existing archive (index -> record) - existing = {} - if archive_path.exists(): - with archive_path.open('r', encoding='utf-8') as f: - for line in f: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - idx = obj.get('index') - if idx is not None: + lock = _get_archive_lock(archive_path) + with lock: + # Load existing archive (index -> record) + existing = {} + if archive_path.exists(): + with archive_path.open('r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + idx = obj.get('index') + if idx is not None: + existing[idx] = obj + + # Add new predictions + new_count = 0 + updated_count = 0 + for jsonl in sorted(predictions_dir.rglob('*.jsonl')): + with jsonl.open('r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + idx = obj.get('index') + if idx is None: + continue + if idx in existing: + updated_count += 1 + else: + new_count += 1 existing[idx] = obj - # Add new predictions - new_count = 0 - updated_count = 0 - for jsonl in sorted(predictions_dir.rglob('*.jsonl')): - with jsonl.open('r', encoding='utf-8') as f: - for line in f: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - idx = obj.get('index') - if idx is None: - continue - if idx in existing: - updated_count += 1 - else: - new_count += 1 - existing[idx] = obj - - # Write back atomically - tmp_path = archive_path.with_suffix('.jsonl.tmp') - with tmp_path.open('w', encoding='utf-8') as f: - for idx in sorted(existing.keys()): - f.write(json.dumps(existing[idx], ensure_ascii=False)) - f.write('\n') - shutil.move(str(tmp_path), str(archive_path)) + # Write back atomically using a unique temp file per thread to avoid + # collisions when multiple runs update the same archive. + tmp_path = archive_path.with_suffix( + f'.jsonl.tmp.{os.getpid()}.{threading.get_ident()}') + with tmp_path.open('w', encoding='utf-8') as f: + for idx in sorted(existing.keys()): + f.write(json.dumps(existing[idx], ensure_ascii=False)) + f.write('\n') + shutil.move(str(tmp_path), str(archive_path)) return archive_path diff --git a/bash/run.py b/bash/run.py index 61c1715..2594454 100644 --- a/bash/run.py +++ b/bash/run.py @@ -31,6 +31,7 @@ Examples: import argparse import json import sys +import threading import time from copy import deepcopy from pathlib import Path @@ -243,6 +244,16 @@ def build_parser(): help='Random seed (default: %(default)s)') parser.add_argument('--batch-size', type=int, default=DEFAULT_BATCH_SIZE, help='Evaluation batch size (default: %(default)s)') + parser.add_argument('--parallel-runs', type=int, default=1, + help='Number of multi-run seeds to execute in parallel ' + '(default: %(default)s). Each run still uses --batch-size ' + 'concurrent requests, so total API concurrency is ' + 'parallel-runs * batch-size.') + parser.add_argument('--parallel-benchmarks', type=int, default=1, + help='Number of different benchmarks to execute in parallel ' + '(default: %(default)s). When >1, each benchmark runs in its ' + 'own thread; completed runs immediately free a slot for the ' + 'next benchmark.') # Decoding / thinking parser.add_argument('--thinking', action='store_true', default=None, @@ -537,7 +548,8 @@ def write_summary(output_dir: str, model_name: str, folder_name: str, def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str, - model_name: str, folder_name: str, benchmark_names: list): + model_name: str, folder_name: str, benchmark_names: list, + summary_lock: threading.Lock = None): """Run a benchmark task and optionally refresh the summary table. ``benchmark_names`` is the running list of canonical benchmark names that @@ -549,6 +561,9 @@ def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str, ``perf_metrics.summary`` and the per-sample predictions into the durable backup files maintained by ``perf_backup.py`` so checkpoint restarts can recover them. + + If ``summary_lock`` is provided, the summary write is protected so multiple + seeds/runs can safely refresh the table as each one finishes. """ dataset_name = task_cfg.datasets[0] work_dir = Path(task_cfg.work_dir) @@ -564,7 +579,11 @@ def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str, backup_after_run(output_dir, dataset_name, model_name, work_dir) if write_summary_flag: - write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names) + if summary_lock is not None: + with summary_lock: + write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names) + else: + write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names) def backup_after_run(output_dir: str, benchmark: str, model_name: str, @@ -710,7 +729,8 @@ def main(): f'max_tokens={DEFAULT_GENERATION_CONFIG["max_tokens"]})') return {'generation_config': deepcopy(DEFAULT_GENERATION_CONFIG)} - def run_one(dataset_name, run_idx=0, benchmark_names=None): + def run_one(dataset_name, run_idx=0, benchmark_names=None, write_summary_flag=True, + summary_lock=None): ds_cfg = get_dataset_config(dataset_name) task_cfg = build_task_config( dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit, @@ -720,8 +740,9 @@ def main(): max_tokens_add=args.max_tokens_add, ) try: - run_and_summarize(task_cfg, args.write_summary, str(model_output_dir), args.model, - folder_name=folder_name, benchmark_names=benchmark_names) + run_and_summarize(task_cfg, write_summary_flag, str(model_output_dir), args.model, + folder_name=folder_name, benchmark_names=benchmark_names, + summary_lock=summary_lock) except Exception as e: print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}') @@ -729,31 +750,89 @@ def main(): # the summary table only includes them, not stale results from earlier runs. benchmark_names = [] + # Build a list of benchmark execution units. Each unit runs one complete + # benchmark (all its multi-runs). This lets us run benchmarks either + # serially or in parallel while keeping per-benchmark run parallelism + # (``--parallel-runs``) intact. + benchmark_units = [] for dataset_name in multi_run: benchmark_names.append(dataset_name) - num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1) - for run_idx in range(num_runs): - print(f"\n{'='*60}") - print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})') - print(f"{'='*60}") - run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names) - + benchmark_units.append((dataset_name, 'multi')) for dataset_name in single_run: benchmark_names.append(dataset_name) - print(f"\n{'='*60}") - print(f'Running: {dataset_name} (seed={args.seed})') - print(f"{'='*60}") - run_one(dataset_name, benchmark_names=benchmark_names) - + benchmark_units.append((dataset_name, 'single')) for dataset_name in agent: benchmark_names.append(dataset_name) + benchmark_units.append((dataset_name, 'agent')) + + def run_benchmark_unit(dataset_name: str, kind: str, summary_lock=None): + """Run one benchmark (all seeds/runs) and return its name.""" + if kind == 'multi': + num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1) + parallel_runs = max(1, args.parallel_runs) + if parallel_runs <= 1: + for run_idx in range(num_runs): + print(f"\n{'='*60}") + print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})') + print(f"{'='*60}") + run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names, + write_summary_flag=True, summary_lock=summary_lock) + else: + print(f"\n{'='*60}") + print(f'Running: {dataset_name} ({num_runs} runs, {parallel_runs} in parallel, ' + f'seed={args.seed}, per-run batch_size={args.batch_size})') + print(f"{'='*60}") + from concurrent.futures import ThreadPoolExecutor, as_completed + + def _run_single(run_idx): + print(f' -> start run {run_idx + 1}/{num_runs}') + run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names, + write_summary_flag=True, summary_lock=summary_lock) + print(f' -> finish run {run_idx + 1}/{num_runs}') + + with ThreadPoolExecutor(max_workers=parallel_runs) as executor: + futures = [executor.submit(_run_single, run_idx) for run_idx in range(num_runs)] + for future in as_completed(futures): + try: + future.result() + except Exception as e: + print(f'ERROR in {dataset_name} parallel run: {e}') + else: + print(f"\n{'='*60}") + print(f'Running: {dataset_name} (seed={args.seed})') + print(f"{'='*60}") + run_one(dataset_name, benchmark_names=benchmark_names, write_summary_flag=True, + summary_lock=summary_lock) + return dataset_name + + # Lock to protect concurrent writes to the summary CSV/Excel when multiple + # seeds/runs or benchmarks finish around the same time. + summary_lock = threading.Lock() + + parallel_benchmarks = max(1, args.parallel_benchmarks) + if parallel_benchmarks <= 1: + for dataset_name, kind in benchmark_units: + run_benchmark_unit(dataset_name, kind, summary_lock=summary_lock) + else: print(f"\n{'='*60}") - print(f'Running: {dataset_name} (seed={args.seed})') + print(f'Running benchmarks in parallel: {len(benchmark_units)} units, ' + f'{parallel_benchmarks} at a time') print(f"{'='*60}") - run_one(dataset_name, benchmark_names=benchmark_names) + from concurrent.futures import ThreadPoolExecutor, as_completed + + with ThreadPoolExecutor(max_workers=parallel_benchmarks) as executor: + futures = [executor.submit(run_benchmark_unit, dataset_name, kind, summary_lock) + for dataset_name, kind in benchmark_units] + for future in as_completed(futures): + try: + dataset_name = future.result() + print(f' -> benchmark finished: {dataset_name}') + except Exception as e: + print(f'ERROR in parallel benchmark: {e}') if args.write_summary: - write_summary(str(model_output_dir), args.model, folder_name, benchmark_names=benchmark_names) + with summary_lock: + write_summary(str(model_output_dir), args.model, folder_name, benchmark_names=benchmark_names) print('\nAll benchmarks done!') diff --git a/config/dpv4-int8_thinking.yaml b/config/dpv4-int8_thinking.yaml index 4570bf4..05b5221 100644 --- a/config/dpv4-int8_thinking.yaml +++ b/config/dpv4-int8_thinking.yaml @@ -4,54 +4,63 @@ gpqa_diamond: top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 hle: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 aime24: generation_config: temperature: 1.0 top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 aime25: generation_config: temperature: 1.0 top_p: 1.0 stream: true max_tokens: 65536 + max_completion_tokens: 64000 mmlu_pro: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 simple_qa: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 arc: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 bbh: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 live_code_bench: generation_config: temperature: 1.0 top_p: 1.0 stream: true max_tokens: 65536 + max_completion_tokens: 64000 dataset_args: subset_list: - release_v6 @@ -61,96 +70,112 @@ aime26: top_p: 1.0 stream: true max_tokens: 65536 + max_completion_tokens: 64000 hmmt26: generation_config: temperature: 1.0 top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 imo_answerbench: generation_config: temperature: 1.0 top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 super_gpqa: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 drop: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 hellaswag: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 mmlu: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 openai_mrcr: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 bigcodebench: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 humaneval: generation_config: temperature: 1.0 top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 gsm8k: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 competition_math: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 cmmlu: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 trivia_qa: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 winogrande: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 longbench_v2: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 + max_completion_tokens: 64000 dataset_args: subset_list: - short @@ -162,6 +187,7 @@ tau2_bench: top_p: 1.0 stream: true max_tokens: 16384 + max_completion_tokens: 64000 dataset_args: extra_params: user_model: deepseek-v4-pro @@ -170,6 +196,7 @@ tau2_bench: generation_config: temperature: 0.0 max_tokens: 4096 + max_completion_tokens: 64000 agent_config: mode: native strategy: react @@ -180,12 +207,14 @@ general_fc: top_p: 1.0 stream: true max_tokens: 4096 + max_completion_tokens: 64000 bfcl_v3: generation_config: temperature: 0.0 top_p: 1.0 stream: true max_tokens: 4096 + max_completion_tokens: 64000 parallel_tool_calls: true dataset_args: extra_params: @@ -197,9 +226,11 @@ swe_bench_verified: top_p: 1.0 stream: true max_tokens: 32768 + max_completion_tokens: 64000 swe_bench_pro: generation_config: temperature: 0.0 top_p: 1.0 stream: true - max_tokens: 32768 \ No newline at end of file + max_tokens: 32768 + max_completion_tokens: 64000 diff --git a/evalscope/evalscope/api/model/generate_config.py b/evalscope/evalscope/api/model/generate_config.py index 530f9dd..b2a8865 100644 --- a/evalscope/evalscope/api/model/generate_config.py +++ b/evalscope/evalscope/api/model/generate_config.py @@ -51,6 +51,9 @@ class GenerateConfig(BaseModel): max_tokens: Optional[int] = Field(default=None) """The maximum number of tokens that can be generated in the completion (default is model specific).""" + max_completion_tokens: Optional[int] = Field(default=None) + """An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens. Used by OpenAI reasoning models and some proxies.""" + top_p: Optional[float] = Field(default=None) """An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass.""" diff --git a/evalscope/evalscope/api/model/model.py b/evalscope/evalscope/api/model/model.py index fcb0c66..ad58f71 100644 --- a/evalscope/evalscope/api/model/model.py +++ b/evalscope/evalscope/api/model/model.py @@ -306,6 +306,9 @@ def get_model_with_task_config(task_config: 'TaskConfig') -> Model: eval_type = task_config.eval_type base_url = task_config.api_url api_key = task_config.api_key + # TaskConfig defaults api_key to 'EMPTY'; treat it as unset so env vars can be used. + if api_key == 'EMPTY': + api_key = None config = task_config.generation_config model_args = task_config.model_args or {} diff --git a/evalscope/evalscope/models/utils/openai.py b/evalscope/evalscope/models/utils/openai.py index 62905b8..632e091 100644 --- a/evalscope/evalscope/models/utils/openai.py +++ b/evalscope/evalscope/models/utils/openai.py @@ -256,6 +256,8 @@ def openai_completion_params(model: str, config: GenerateConfig, tools: bool) -> params['timeout'] = config.timeout if config.max_tokens is not None: params['max_tokens'] = config.max_tokens + if config.max_completion_tokens is not None: + params['max_completion_tokens'] = config.max_completion_tokens if config.frequency_penalty is not None: params['frequency_penalty'] = config.frequency_penalty if config.stop_seqs is not None: @@ -822,6 +824,9 @@ def collect_stream_response( continue collected_chunks.append(chunk) for choice in chunk.choices: + # Some proxies (e.g. Vectron) omit choice.index; default to 0. + choice_index = choice.index if choice.index is not None else 0 + # Detect first meaningful content chunk for TTFT has_content = ((choice.delta.content is not None and choice.delta.content != '') or ( hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None @@ -832,11 +837,11 @@ def collect_stream_response( # Handle reasoning content if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None: - collected_reasoning[choice.index].append(choice.delta.reasoning_content) + collected_reasoning[choice_index].append(choice.delta.reasoning_content) # Handle regular content if choice.delta.content is not None: - collected_messages[choice.index].append(choice.delta.content) + collected_messages[choice_index].append(choice.delta.content) # Handle tool calls if hasattr(choice.delta, 'tool_calls') and choice.delta.tool_calls: @@ -844,8 +849,8 @@ def collect_stream_response( tool_id = tool_call.index # Initialize tool call if not present - if tool_id not in collected_tool_calls[choice.index]: - collected_tool_calls[choice.index][tool_id] = { + if tool_id not in collected_tool_calls[choice_index]: + collected_tool_calls[choice_index][tool_id] = { 'id': tool_call.id if hasattr(tool_call, 'id') and tool_call.id else None, 'type': tool_call.type if hasattr(tool_call, 'type') and tool_call.type else None, 'function': { @@ -857,15 +862,15 @@ def collect_stream_response( # Update tool call with new chunks if hasattr(tool_call, 'function'): if hasattr(tool_call.function, 'name') and tool_call.function.name: - collected_tool_calls[choice.index][tool_id]['function']['name'] = tool_call.function.name + collected_tool_calls[choice_index][tool_id]['function']['name'] = tool_call.function.name if hasattr(tool_call.function, 'arguments') and tool_call.function.arguments: - collected_tool_calls[choice.index + collected_tool_calls[choice_index ][tool_id]['function']['arguments'] += tool_call.function.arguments # Update ID if it was received later if hasattr(tool_call, 'id') and tool_call.id: - collected_tool_calls[choice.index][tool_id]['id'] = tool_call.id + collected_tool_calls[choice_index][tool_id]['id'] = tool_call.id # Get all unique choice indices from all collections all_indices = set(collected_messages.keys()) | set(collected_reasoning.keys()) | set(collected_tool_calls.keys()) @@ -885,9 +890,11 @@ def collect_stream_response( # use the finish_reason from the last chunk that generated this choice finish_reason = None for chunk in reversed(collected_chunks): - if chunk.choices and chunk.choices[0].index == index: - finish_reason = chunk.choices[0].finish_reason - break + if chunk.choices: + chunk_index = chunk.choices[0].index if chunk.choices[0].index is not None else 0 + if chunk_index == index: + finish_reason = chunk.choices[0].finish_reason + break message_kwargs = {'role': 'assistant', 'content': full_reply_content} @@ -950,6 +957,9 @@ async def async_collect_stream_response( continue collected_chunks.append(chunk) for choice in chunk.choices: + # Some proxies (e.g. Vectron) omit choice.index; default to 0. + choice_index = choice.index if choice.index is not None else 0 + # Detect first meaningful content chunk for TTFT has_content = ((choice.delta.content is not None and choice.delta.content != '') or ( hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None @@ -960,11 +970,11 @@ async def async_collect_stream_response( # Handle reasoning content if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None: - collected_reasoning[choice.index].append(choice.delta.reasoning_content) + collected_reasoning[choice_index].append(choice.delta.reasoning_content) # Handle regular content if choice.delta.content is not None: - collected_messages[choice.index].append(choice.delta.content) + collected_messages[choice_index].append(choice.delta.content) # Handle tool calls if hasattr(choice.delta, 'tool_calls') and choice.delta.tool_calls: @@ -972,8 +982,8 @@ async def async_collect_stream_response( tool_id = tool_call.index # Initialize tool call if not present - if tool_id not in collected_tool_calls[choice.index]: - collected_tool_calls[choice.index][tool_id] = { + if tool_id not in collected_tool_calls[choice_index]: + collected_tool_calls[choice_index][tool_id] = { 'id': tool_call.id if hasattr(tool_call, 'id') and tool_call.id else None, 'type': tool_call.type if hasattr(tool_call, 'type') and tool_call.type else None, 'function': { @@ -985,15 +995,15 @@ async def async_collect_stream_response( # Update tool call with new chunks if hasattr(tool_call, 'function'): if hasattr(tool_call.function, 'name') and tool_call.function.name: - collected_tool_calls[choice.index][tool_id]['function']['name'] = tool_call.function.name + collected_tool_calls[choice_index][tool_id]['function']['name'] = tool_call.function.name if hasattr(tool_call.function, 'arguments') and tool_call.function.arguments: - collected_tool_calls[choice.index + collected_tool_calls[choice_index ][tool_id]['function']['arguments'] += tool_call.function.arguments # Update ID if it was received later if hasattr(tool_call, 'id') and tool_call.id: - collected_tool_calls[choice.index][tool_id]['id'] = tool_call.id + collected_tool_calls[choice_index][tool_id]['id'] = tool_call.id # Get all unique choice indices from all collections all_indices = set(collected_messages.keys()) | set(collected_reasoning.keys()) | set(collected_tool_calls.keys()) @@ -1013,9 +1023,11 @@ async def async_collect_stream_response( # use the finish_reason from the last chunk that generated this choice finish_reason = None for chunk in reversed(collected_chunks): - if chunk.choices and chunk.choices[0].index == index: - finish_reason = chunk.choices[0].finish_reason - break + if chunk.choices: + chunk_index = chunk.choices[0].index if chunk.choices[0].index is not None else 0 + if chunk_index == index: + finish_reason = chunk.choices[0].finish_reason + break message_kwargs = {'role': 'assistant', 'content': full_reply_content} diff --git a/webui/server.py b/webui/server.py index 5e1dba9..3d0e0c9 100644 --- a/webui/server.py +++ b/webui/server.py @@ -408,6 +408,8 @@ async def start_job(req: LaunchRequest) -> JobRecord: env['PYTHONUNBUFFERED'] = '1' if req.api_key and req.api_key != 'EMPTY': env['OPENAI_API_KEY'] = req.api_key + # EvalScope's openai_api backend reads EVALSCOPE_API_KEY. + env['EVALSCOPE_API_KEY'] = req.api_key try: proc = await asyncio.create_subprocess_exec(