diff --git a/bash/run.py b/bash/run.py index 7d20fa7..6fc8479 100644 --- a/bash/run.py +++ b/bash/run.py @@ -40,6 +40,9 @@ import yaml from evalscope import run_task, TaskConfig from evalscope.api.agent import NativeAgentConfig from evalscope.config import SandboxTaskConfig +from evalscope.utils.logger import get_logger + +logger = get_logger() SCRIPT_DIR = Path(__file__).parent.resolve() PROJECT_ROOT = SCRIPT_DIR.parent @@ -281,13 +284,43 @@ _TOKENIZER = None def get_tokenizer(tokenizer_path: str): global _TOKENIZER - if _TOKENIZER is None: - from transformers import AutoTokenizer + if _TOKENIZER is not None: + return _TOKENIZER + + from transformers import AutoTokenizer + import os + + # Try local path first + if tokenizer_path and os.path.exists(tokenizer_path): try: _TOKENIZER = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True) - except Exception: - _TOKENIZER = AutoTokenizer.from_pretrained('deepseek-ai/DeepSeek-V4-Flash', trust_remote_code=True) - return _TOKENIZER + logger.info(f'Loaded tokenizer from local path: {tokenizer_path}') + return _TOKENIZER + except Exception as e: + logger.warning(f'Failed to load tokenizer from local path {tokenizer_path}: {e}') + + # If a local path was explicitly provided but does not exist, warn clearly + if tokenizer_path and tokenizer_path != '/data1/models/DeepSeek-V4-Flash-INT8': + logger.warning(f'Local tokenizer path does not exist: {tokenizer_path}. ' + f'Will try to download from model hub.') + + # Fallback: try model hub. Respect ModelScope hub if configured. + fallback_model = 'deepseek-ai/DeepSeek-V4-Flash' + hub_source = 'ModelScope' if os.environ.get('USE_MODELSCOPE_HUB') == '1' else 'HuggingFace' + logger.info(f'Trying to load tokenizer from {hub_source}: {fallback_model}') + try: + _TOKENIZER = AutoTokenizer.from_pretrained(fallback_model, trust_remote_code=True) + logger.info(f'Loaded tokenizer from {hub_source}: {fallback_model}') + return _TOKENIZER + except Exception as e: + raise RuntimeError( + f'Failed to load tokenizer.\n' + f' Local path: {tokenizer_path} (exists: {os.path.exists(tokenizer_path) if tokenizer_path else False})\n' + f' {hub_source} fallback: {fallback_model}\n' + f' Error: {e}\n' + f' Hint: mount the tokenizer to the container and set --tokenizer-path correctly, ' + f'or set USE_MODELSCOPE_HUB=1 and HF_ENDPOINT=https://hf-mirror.com if downloading.' + ) from e def truncate_middle(text: str, max_tokens: int, tokenizer_path: str) -> str: