chore: update evalscope runner, thinking config and webui

bash/case/DP4-flash-int8-thinking-official.sh

@@ -1,6 +1,2 @@
-python bash/run.py --datasets swe_bench_verified --folder-name DP4-flash-int8-not-thinking
+
-python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add0 --thinking   --max-tokens-add 0
+python bash/run.py --datasets live_code_bench  --folder-name DP4-flash-int8-thinking --thinking  --config /data1/sora/evalscope/config/dpv4-int8_thinking.yaml
-python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add16k --thinking   --max-tokens-add 16384
-python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add32k --thinking   --max-tokens-add 32768
-python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add64k --thinking   --max-tokens-add 65536
-python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add128k --thinking   --max-tokens-add 131072

bash/collect_results.py

@@ -206,17 +206,23 @@ def find_archive_predictions(output_dir: Path, benchmark: str, model_name: str):
 def find_all_predictions(output_dir: Path, benchmark: str, model_name: str):
     """Find all predictions JSONL files for a benchmark/model.
 
-    The durable ``predictions_archive/<benchmark>__<model>.jsonl`` (if
+    For single-seed runs the durable ``predictions_archive`` is preferred so
-    present) is returned **first** so its deduplicated per-sample records
+    breakpoint-resume does not lose completed samples.  When multiple seed/run
-    dominate any smaller predictions that a fresh run may have written.
+    directories exist (multi-run benchmarks) we aggregate from each run
+    separately and skip the archive, because the archive only keeps the latest
+    record per ``index`` and would otherwise collide with one of the runs.
     """
-    files = list(find_archive_predictions(output_dir, benchmark, model_name))
     bench_dir = output_dir / benchmark
-    if not bench_dir.exists():
+    seed_dirs = []
-        return files
+    if bench_dir.exists():
-    for seed_dir in sorted(bench_dir.iterdir()):
+        seed_dirs = sorted([p for p in bench_dir.iterdir() if p.is_dir()])
-        if not seed_dir.is_dir():
+
-            continue
+    files = []
+    # Only rely on the archive for single-run / resume scenarios.
+    if len(seed_dirs) <= 1:
+        files = list(find_archive_predictions(output_dir, benchmark, model_name))
+
+    for seed_dir in seed_dirs:
         pred_dir = seed_dir / 'predictions'
         if pred_dir.exists():
             files.extend(sorted(pred_dir.rglob('*.jsonl')))
@@ -283,11 +289,14 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str):
 
     # Aggregate raw prediction perf metrics across all seeds/runs.
     # This fixes the breakpoint-resume issue where cumulative stats are reset.
-    # Predictions are deduplicated by sample `index` so the archive (which
+    # Deduplicate by (run, sample `index`) so that:
-    # spans every run) and the latest ``predictions/*.jsonl`` don't double
+    #   1. multiple prediction jsonl files inside the same run directory
-    # count the same sample.
+    #      (e.g. `<benchmark>__<model>.jsonl` and `<benchmark>_<subset>.jsonl`)
+    #      do not double-count the same sample;
+    #   2. each seed/run still contributes its own predictions for multi-run
+    #      benchmarks, so total sample count is sum(runs).
     pred_files = find_all_predictions(output_dir, benchmark, model_name)
-    seen_indexes = set()
+    seen_keys = set()
     latencies = []
     ttfts = []
     tpots = []
@@ -295,30 +304,42 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str):
     output_tokens = []
     sample_indexes = []
     for pf in pred_files:
+        # ``run_key`` is the seed/run directory name, or the archive filename
+        # for archive-only scenarios.
+        run_key = pf.name
+        parts = pf.parts
+        if 'predictions_archive' not in parts:
+            for i, part in enumerate(parts):
+                if part == 'predictions' and i > 0:
+                    run_key = parts[i - 1]
+                    break
         for obj in read_predictions_with_index(pf):
             idx = obj['index']
-            if idx is None or idx in seen_indexes:
+            key = (run_key, idx)
-                # Still keep perf data even when we lack an index, so older
+            if idx is None:
+                # Keep perf data even when we lack an index, so older
                 # benchmark files without `index` don't get dropped.
                 pm = obj['perf_metrics']
+            elif key in seen_keys:
+                continue
             else:
-                seen_indexes.add(idx)
+                seen_keys.add(key)
                 sample_indexes.append(idx)
                 pm = obj['perf_metrics']
             if pm is None:
                 continue
-            if 'latency' in pm:
+            if pm.get('latency') is not None:
                 latencies.append(float(pm['latency']))
-            if 'ttft' in pm:
+            if pm.get('ttft') is not None:
                 ttfts.append(float(pm['ttft']))
-            if 'tpot' in pm:
+            if pm.get('tpot') is not None:
                 tpots.append(float(pm['tpot']))
 
             itok = pm.get('input_tokens')
             otok = pm.get('output_tokens')
-            if itok is None and 'usage' in pm:
+            if (itok is None or otok is None) and 'usage' in pm:
-                itok = pm['usage'].get('input_tokens')
+                itok = pm['usage'].get('input_tokens') if itok is None else itok
-                otok = pm['usage'].get('output_tokens')
+                otok = pm['usage'].get('output_tokens') if otok is None else otok
             if itok is not None:
                 input_tokens.append(int(itok))
             if otok is not None:

bash/perf_backup.py

@@ -20,7 +20,9 @@ breakpoint-resume issue can be reconstructed:
 """
 
 import json
+import os
 import shutil
+import threading
 from datetime import datetime
 from pathlib import Path
 
@@ -86,6 +88,20 @@ def backup_perf_stats(output_dir: Path, benchmark: str, model_name: str,
     return perf_stats_path
 
 
+# Per-archive lock to prevent concurrent read-modify-write races when multiple
+# seeds of the same benchmark finish at the same time.
+_archive_locks: dict = {}
+_archive_locks_lock = threading.Lock()
+
+
+def _get_archive_lock(archive_path: Path) -> threading.Lock:
+    key = str(archive_path)
+    with _archive_locks_lock:
+        if key not in _archive_locks:
+            _archive_locks[key] = threading.Lock()
+        return _archive_locks[key]
+
+
 def archive_predictions(output_dir: Path, benchmark: str, model_name: str,
                          predictions_dir: Path) -> Path:
     """Append the latest predictions into the durable archive.
@@ -95,6 +111,8 @@ def archive_predictions(output_dir: Path, benchmark: str, model_name: str,
     the same ``index`` was already archived with a different perf payload,
     the newer record wins.
 
+    This function is thread-safe for concurrent updates to the same archive.
+
     Returns the archive path.
     """
     _, archive_path = get_backup_paths(Path(output_dir), benchmark, model_name)
@@ -103,51 +121,55 @@ def archive_predictions(output_dir: Path, benchmark: str, model_name: str,
     if not predictions_dir or not predictions_dir.exists():
         return archive_path
 
-    # Load existing archive (index -> record)
+    lock = _get_archive_lock(archive_path)
-    existing = {}
+    with lock:
-    if archive_path.exists():
+        # Load existing archive (index -> record)
-        with archive_path.open('r', encoding='utf-8') as f:
+        existing = {}
-            for line in f:
+        if archive_path.exists():
-                line = line.strip()
+            with archive_path.open('r', encoding='utf-8') as f:
-                if not line:
+                for line in f:
-                    continue
+                    line = line.strip()
-                try:
+                    if not line:
-                    obj = json.loads(line)
+                        continue
-                except json.JSONDecodeError:
+                    try:
-                    continue
+                        obj = json.loads(line)
-                idx = obj.get('index')
+                    except json.JSONDecodeError:
-                if idx is not None:
+                        continue
+                    idx = obj.get('index')
+                    if idx is not None:
+                        existing[idx] = obj
+
+        # Add new predictions
+        new_count = 0
+        updated_count = 0
+        for jsonl in sorted(predictions_dir.rglob('*.jsonl')):
+            with jsonl.open('r', encoding='utf-8') as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        obj = json.loads(line)
+                    except json.JSONDecodeError:
+                        continue
+                    idx = obj.get('index')
+                    if idx is None:
+                        continue
+                    if idx in existing:
+                        updated_count += 1
+                    else:
+                        new_count += 1
                     existing[idx] = obj
 
-    # Add new predictions
+        # Write back atomically using a unique temp file per thread to avoid
-    new_count = 0
+        # collisions when multiple runs update the same archive.
-    updated_count = 0
+        tmp_path = archive_path.with_suffix(
-    for jsonl in sorted(predictions_dir.rglob('*.jsonl')):
+            f'.jsonl.tmp.{os.getpid()}.{threading.get_ident()}')
-        with jsonl.open('r', encoding='utf-8') as f:
+        with tmp_path.open('w', encoding='utf-8') as f:
-            for line in f:
+            for idx in sorted(existing.keys()):
-                line = line.strip()
+                f.write(json.dumps(existing[idx], ensure_ascii=False))
-                if not line:
+                f.write('\n')
-                    continue
+        shutil.move(str(tmp_path), str(archive_path))
-                try:
-                    obj = json.loads(line)
-                except json.JSONDecodeError:
-                    continue
-                idx = obj.get('index')
-                if idx is None:
-                    continue
-                if idx in existing:
-                    updated_count += 1
-                else:
-                    new_count += 1
-                existing[idx] = obj
-
-    # Write back atomically
-    tmp_path = archive_path.with_suffix('.jsonl.tmp')
-    with tmp_path.open('w', encoding='utf-8') as f:
-        for idx in sorted(existing.keys()):
-            f.write(json.dumps(existing[idx], ensure_ascii=False))
-            f.write('\n')
-    shutil.move(str(tmp_path), str(archive_path))
 
     return archive_path
 

bash/run.py

@@ -31,6 +31,7 @@ Examples:
 import argparse
 import json
 import sys
+import threading
 import time
 from copy import deepcopy
 from pathlib import Path
@@ -243,6 +244,16 @@ def build_parser():
                         help='Random seed (default: %(default)s)')
     parser.add_argument('--batch-size', type=int, default=DEFAULT_BATCH_SIZE,
                         help='Evaluation batch size (default: %(default)s)')
+    parser.add_argument('--parallel-runs', type=int, default=1,
+                        help='Number of multi-run seeds to execute in parallel '
+                             '(default: %(default)s). Each run still uses --batch-size '
+                             'concurrent requests, so total API concurrency is '
+                             'parallel-runs * batch-size.')
+    parser.add_argument('--parallel-benchmarks', type=int, default=1,
+                        help='Number of different benchmarks to execute in parallel '
+                             '(default: %(default)s). When >1, each benchmark runs in its '
+                             'own thread; completed runs immediately free a slot for the '
+                             'next benchmark.')
 
     # Decoding / thinking
     parser.add_argument('--thinking', action='store_true', default=None,
@@ -537,7 +548,8 @@ def write_summary(output_dir: str, model_name: str, folder_name: str,
 
 
 def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str,
-                       model_name: str, folder_name: str, benchmark_names: list):
+                       model_name: str, folder_name: str, benchmark_names: list,
+                       summary_lock: threading.Lock = None):
     """Run a benchmark task and optionally refresh the summary table.
 
     ``benchmark_names`` is the running list of canonical benchmark names that
@@ -549,6 +561,9 @@ def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str,
     ``perf_metrics.summary`` and the per-sample predictions into the durable
     backup files maintained by ``perf_backup.py`` so checkpoint restarts can
     recover them.
+
+    If ``summary_lock`` is provided, the summary write is protected so multiple
+    seeds/runs can safely refresh the table as each one finishes.
     """
     dataset_name = task_cfg.datasets[0]
     work_dir = Path(task_cfg.work_dir)
@@ -564,7 +579,11 @@ def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str,
 
     backup_after_run(output_dir, dataset_name, model_name, work_dir)
     if write_summary_flag:
-        write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names)
+        if summary_lock is not None:
+            with summary_lock:
+                write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names)
+        else:
+            write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names)
 
 
 def backup_after_run(output_dir: str, benchmark: str, model_name: str,
@@ -710,7 +729,8 @@ def main():
               f'max_tokens={DEFAULT_GENERATION_CONFIG["max_tokens"]})')
         return {'generation_config': deepcopy(DEFAULT_GENERATION_CONFIG)}
 
-    def run_one(dataset_name, run_idx=0, benchmark_names=None):
+    def run_one(dataset_name, run_idx=0, benchmark_names=None, write_summary_flag=True,
+                summary_lock=None):
         ds_cfg = get_dataset_config(dataset_name)
         task_cfg = build_task_config(
             dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
@@ -720,8 +740,9 @@ def main():
             max_tokens_add=args.max_tokens_add,
         )
         try:
-            run_and_summarize(task_cfg, args.write_summary, str(model_output_dir), args.model,
+            run_and_summarize(task_cfg, write_summary_flag, str(model_output_dir), args.model,
-                              folder_name=folder_name, benchmark_names=benchmark_names)
+                              folder_name=folder_name, benchmark_names=benchmark_names,
+                              summary_lock=summary_lock)
         except Exception as e:
             print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}')
 
@@ -729,31 +750,89 @@ def main():
     # the summary table only includes them, not stale results from earlier runs.
     benchmark_names = []
 
+    # Build a list of benchmark execution units.  Each unit runs one complete
+    # benchmark (all its multi-runs).  This lets us run benchmarks either
+    # serially or in parallel while keeping per-benchmark run parallelism
+    # (``--parallel-runs``) intact.
+    benchmark_units = []
     for dataset_name in multi_run:
         benchmark_names.append(dataset_name)
-        num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1)
+        benchmark_units.append((dataset_name, 'multi'))
-        for run_idx in range(num_runs):
-            print(f"\n{'='*60}")
-            print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})')
-            print(f"{'='*60}")
-            run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names)
-
     for dataset_name in single_run:
         benchmark_names.append(dataset_name)
-        print(f"\n{'='*60}")
+        benchmark_units.append((dataset_name, 'single'))
-        print(f'Running: {dataset_name} (seed={args.seed})')
-        print(f"{'='*60}")
-        run_one(dataset_name, benchmark_names=benchmark_names)
-
     for dataset_name in agent:
         benchmark_names.append(dataset_name)
+        benchmark_units.append((dataset_name, 'agent'))
+
+    def run_benchmark_unit(dataset_name: str, kind: str, summary_lock=None):
+        """Run one benchmark (all seeds/runs) and return its name."""
+        if kind == 'multi':
+            num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1)
+            parallel_runs = max(1, args.parallel_runs)
+            if parallel_runs <= 1:
+                for run_idx in range(num_runs):
+                    print(f"\n{'='*60}")
+                    print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})')
+                    print(f"{'='*60}")
+                    run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names,
+                            write_summary_flag=True, summary_lock=summary_lock)
+            else:
+                print(f"\n{'='*60}")
+                print(f'Running: {dataset_name} ({num_runs} runs, {parallel_runs} in parallel, '
+                      f'seed={args.seed}, per-run batch_size={args.batch_size})')
+                print(f"{'='*60}")
+                from concurrent.futures import ThreadPoolExecutor, as_completed
+
+                def _run_single(run_idx):
+                    print(f'  -> start run {run_idx + 1}/{num_runs}')
+                    run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names,
+                            write_summary_flag=True, summary_lock=summary_lock)
+                    print(f'  -> finish run {run_idx + 1}/{num_runs}')
+
+                with ThreadPoolExecutor(max_workers=parallel_runs) as executor:
+                    futures = [executor.submit(_run_single, run_idx) for run_idx in range(num_runs)]
+                    for future in as_completed(futures):
+                        try:
+                            future.result()
+                        except Exception as e:
+                            print(f'ERROR in {dataset_name} parallel run: {e}')
+        else:
+            print(f"\n{'='*60}")
+            print(f'Running: {dataset_name} (seed={args.seed})')
+            print(f"{'='*60}")
+            run_one(dataset_name, benchmark_names=benchmark_names, write_summary_flag=True,
+                    summary_lock=summary_lock)
+        return dataset_name
+
+    # Lock to protect concurrent writes to the summary CSV/Excel when multiple
+    # seeds/runs or benchmarks finish around the same time.
+    summary_lock = threading.Lock()
+
+    parallel_benchmarks = max(1, args.parallel_benchmarks)
+    if parallel_benchmarks <= 1:
+        for dataset_name, kind in benchmark_units:
+            run_benchmark_unit(dataset_name, kind, summary_lock=summary_lock)
+    else:
         print(f"\n{'='*60}")
-        print(f'Running: {dataset_name} (seed={args.seed})')
+        print(f'Running benchmarks in parallel: {len(benchmark_units)} units, '
+              f'{parallel_benchmarks} at a time')
         print(f"{'='*60}")
-        run_one(dataset_name, benchmark_names=benchmark_names)
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+
+        with ThreadPoolExecutor(max_workers=parallel_benchmarks) as executor:
+            futures = [executor.submit(run_benchmark_unit, dataset_name, kind, summary_lock)
+                       for dataset_name, kind in benchmark_units]
+            for future in as_completed(futures):
+                try:
+                    dataset_name = future.result()
+                    print(f'  -> benchmark finished: {dataset_name}')
+                except Exception as e:
+                    print(f'ERROR in parallel benchmark: {e}')
 
     if args.write_summary:
-        write_summary(str(model_output_dir), args.model, folder_name, benchmark_names=benchmark_names)
+        with summary_lock:
+            write_summary(str(model_output_dir), args.model, folder_name, benchmark_names=benchmark_names)
     print('\nAll benchmarks done!')
 
 

config/dpv4-int8_thinking.yaml

@@ -4,54 +4,63 @@ gpqa_diamond:
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 hle:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 aime24:
   generation_config:
     temperature: 1.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 aime25:
   generation_config:
     temperature: 1.0
     top_p: 1.0
     stream: true
     max_tokens: 65536
+    max_completion_tokens: 64000
 mmlu_pro:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 simple_qa:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 arc:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 bbh:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 live_code_bench:
   generation_config:
     temperature: 1.0
     top_p: 1.0
     stream: true
     max_tokens: 65536
+    max_completion_tokens: 64000
   dataset_args:
     subset_list:
     - release_v6
@@ -61,96 +70,112 @@ aime26:
     top_p: 1.0
     stream: true
     max_tokens: 65536
+    max_completion_tokens: 64000
 hmmt26:
   generation_config:
     temperature: 1.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 imo_answerbench:
   generation_config:
     temperature: 1.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 super_gpqa:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 drop:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 hellaswag:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 mmlu:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 openai_mrcr:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 bigcodebench:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 humaneval:
   generation_config:
     temperature: 1.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 gsm8k:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 competition_math:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 cmmlu:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 trivia_qa:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 winogrande:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
 longbench_v2:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 8192
+    max_completion_tokens: 64000
   dataset_args:
     subset_list:
     - short
@@ -162,6 +187,7 @@ tau2_bench:
     top_p: 1.0
     stream: true
     max_tokens: 16384
+    max_completion_tokens: 64000
   dataset_args:
     extra_params:
       user_model: deepseek-v4-pro
@@ -170,6 +196,7 @@ tau2_bench:
       generation_config:
         temperature: 0.0
         max_tokens: 4096
+        max_completion_tokens: 64000
   agent_config:
     mode: native
     strategy: react
@@ -180,12 +207,14 @@ general_fc:
     top_p: 1.0
     stream: true
     max_tokens: 4096
+    max_completion_tokens: 64000
 bfcl_v3:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 4096
+    max_completion_tokens: 64000
     parallel_tool_calls: true
   dataset_args:
     extra_params:
@@ -197,9 +226,11 @@ swe_bench_verified:
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000
 swe_bench_pro:
   generation_config:
     temperature: 0.0
     top_p: 1.0
     stream: true
     max_tokens: 32768
+    max_completion_tokens: 64000

evalscope/evalscope/api/model/generate_config.py

@@ -51,6 +51,9 @@ class GenerateConfig(BaseModel):
     max_tokens: Optional[int] = Field(default=None)
     """The maximum number of tokens that can be generated in the completion (default is model specific)."""
 
+    max_completion_tokens: Optional[int] = Field(default=None)
+    """An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens. Used by OpenAI reasoning models and some proxies."""
+
     top_p: Optional[float] = Field(default=None)
     """An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass."""
 

evalscope/evalscope/api/model/model.py

@@ -306,6 +306,9 @@ def get_model_with_task_config(task_config: 'TaskConfig') -> Model:
     eval_type = task_config.eval_type
     base_url = task_config.api_url
     api_key = task_config.api_key
+    # TaskConfig defaults api_key to 'EMPTY'; treat it as unset so env vars can be used.
+    if api_key == 'EMPTY':
+        api_key = None
     config = task_config.generation_config
     model_args = task_config.model_args or {}
 

evalscope/evalscope/models/utils/openai.py

@@ -256,6 +256,8 @@ def openai_completion_params(model: str, config: GenerateConfig, tools: bool) ->
         params['timeout'] = config.timeout
     if config.max_tokens is not None:
         params['max_tokens'] = config.max_tokens
+    if config.max_completion_tokens is not None:
+        params['max_completion_tokens'] = config.max_completion_tokens
     if config.frequency_penalty is not None:
         params['frequency_penalty'] = config.frequency_penalty
     if config.stop_seqs is not None:
@@ -822,6 +824,9 @@ def collect_stream_response(
             continue
         collected_chunks.append(chunk)
         for choice in chunk.choices:
+            # Some proxies (e.g. Vectron) omit choice.index; default to 0.
+            choice_index = choice.index if choice.index is not None else 0
+
             # Detect first meaningful content chunk for TTFT
             has_content = ((choice.delta.content is not None and choice.delta.content != '') or (
                 hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None
@@ -832,11 +837,11 @@ def collect_stream_response(
 
             # Handle reasoning content
             if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None:
-                collected_reasoning[choice.index].append(choice.delta.reasoning_content)
+                collected_reasoning[choice_index].append(choice.delta.reasoning_content)
 
             # Handle regular content
             if choice.delta.content is not None:
-                collected_messages[choice.index].append(choice.delta.content)
+                collected_messages[choice_index].append(choice.delta.content)
 
             # Handle tool calls
             if hasattr(choice.delta, 'tool_calls') and choice.delta.tool_calls:
@@ -844,8 +849,8 @@ def collect_stream_response(
                     tool_id = tool_call.index
 
                     # Initialize tool call if not present
-                    if tool_id not in collected_tool_calls[choice.index]:
+                    if tool_id not in collected_tool_calls[choice_index]:
-                        collected_tool_calls[choice.index][tool_id] = {
+                        collected_tool_calls[choice_index][tool_id] = {
                             'id': tool_call.id if hasattr(tool_call, 'id') and tool_call.id else None,
                             'type': tool_call.type if hasattr(tool_call, 'type') and tool_call.type else None,
                             'function': {
@@ -857,15 +862,15 @@ def collect_stream_response(
                     # Update tool call with new chunks
                     if hasattr(tool_call, 'function'):
                         if hasattr(tool_call.function, 'name') and tool_call.function.name:
-                            collected_tool_calls[choice.index][tool_id]['function']['name'] = tool_call.function.name
+                            collected_tool_calls[choice_index][tool_id]['function']['name'] = tool_call.function.name
 
                         if hasattr(tool_call.function, 'arguments') and tool_call.function.arguments:
-                            collected_tool_calls[choice.index
+                            collected_tool_calls[choice_index
                                                  ][tool_id]['function']['arguments'] += tool_call.function.arguments
 
                     # Update ID if it was received later
                     if hasattr(tool_call, 'id') and tool_call.id:
-                        collected_tool_calls[choice.index][tool_id]['id'] = tool_call.id
+                        collected_tool_calls[choice_index][tool_id]['id'] = tool_call.id
 
     # Get all unique choice indices from all collections
     all_indices = set(collected_messages.keys()) | set(collected_reasoning.keys()) | set(collected_tool_calls.keys())
@@ -885,9 +890,11 @@ def collect_stream_response(
         # use the finish_reason from the last chunk that generated this choice
         finish_reason = None
         for chunk in reversed(collected_chunks):
-            if chunk.choices and chunk.choices[0].index == index:
+            if chunk.choices:
-                finish_reason = chunk.choices[0].finish_reason
+                chunk_index = chunk.choices[0].index if chunk.choices[0].index is not None else 0
-                break
+                if chunk_index == index:
+                    finish_reason = chunk.choices[0].finish_reason
+                    break
 
         message_kwargs = {'role': 'assistant', 'content': full_reply_content}
 
@@ -950,6 +957,9 @@ async def async_collect_stream_response(
             continue
         collected_chunks.append(chunk)
         for choice in chunk.choices:
+            # Some proxies (e.g. Vectron) omit choice.index; default to 0.
+            choice_index = choice.index if choice.index is not None else 0
+
             # Detect first meaningful content chunk for TTFT
             has_content = ((choice.delta.content is not None and choice.delta.content != '') or (
                 hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None
@@ -960,11 +970,11 @@ async def async_collect_stream_response(
 
             # Handle reasoning content
             if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None:
-                collected_reasoning[choice.index].append(choice.delta.reasoning_content)
+                collected_reasoning[choice_index].append(choice.delta.reasoning_content)
 
             # Handle regular content
             if choice.delta.content is not None:
-                collected_messages[choice.index].append(choice.delta.content)
+                collected_messages[choice_index].append(choice.delta.content)
 
             # Handle tool calls
             if hasattr(choice.delta, 'tool_calls') and choice.delta.tool_calls:
@@ -972,8 +982,8 @@ async def async_collect_stream_response(
                     tool_id = tool_call.index
 
                     # Initialize tool call if not present
-                    if tool_id not in collected_tool_calls[choice.index]:
+                    if tool_id not in collected_tool_calls[choice_index]:
-                        collected_tool_calls[choice.index][tool_id] = {
+                        collected_tool_calls[choice_index][tool_id] = {
                             'id': tool_call.id if hasattr(tool_call, 'id') and tool_call.id else None,
                             'type': tool_call.type if hasattr(tool_call, 'type') and tool_call.type else None,
                             'function': {
@@ -985,15 +995,15 @@ async def async_collect_stream_response(
                     # Update tool call with new chunks
                     if hasattr(tool_call, 'function'):
                         if hasattr(tool_call.function, 'name') and tool_call.function.name:
-                            collected_tool_calls[choice.index][tool_id]['function']['name'] = tool_call.function.name
+                            collected_tool_calls[choice_index][tool_id]['function']['name'] = tool_call.function.name
 
                         if hasattr(tool_call.function, 'arguments') and tool_call.function.arguments:
-                            collected_tool_calls[choice.index
+                            collected_tool_calls[choice_index
                                                  ][tool_id]['function']['arguments'] += tool_call.function.arguments
 
                     # Update ID if it was received later
                     if hasattr(tool_call, 'id') and tool_call.id:
-                        collected_tool_calls[choice.index][tool_id]['id'] = tool_call.id
+                        collected_tool_calls[choice_index][tool_id]['id'] = tool_call.id
 
     # Get all unique choice indices from all collections
     all_indices = set(collected_messages.keys()) | set(collected_reasoning.keys()) | set(collected_tool_calls.keys())
@@ -1013,9 +1023,11 @@ async def async_collect_stream_response(
         # use the finish_reason from the last chunk that generated this choice
         finish_reason = None
         for chunk in reversed(collected_chunks):
-            if chunk.choices and chunk.choices[0].index == index:
+            if chunk.choices:
-                finish_reason = chunk.choices[0].finish_reason
+                chunk_index = chunk.choices[0].index if chunk.choices[0].index is not None else 0
-                break
+                if chunk_index == index:
+                    finish_reason = chunk.choices[0].finish_reason
+                    break
 
         message_kwargs = {'role': 'assistant', 'content': full_reply_content}
 

webui/server.py

@@ -408,6 +408,8 @@ async def start_job(req: LaunchRequest) -> JobRecord:
         env['PYTHONUNBUFFERED'] = '1'
         if req.api_key and req.api_key != 'EMPTY':
             env['OPENAI_API_KEY'] = req.api_key
+            # EvalScope's openai_api backend reads EVALSCOPE_API_KEY.
+            env['EVALSCOPE_API_KEY'] = req.api_key
 
         try:
             proc = await asyncio.create_subprocess_exec(