Comparison-driven fixes: MCQ choices in prompt + letter contract, Answer: suffix for QA, markdown answer cleaning, trivia_qa answer_phrase priority, DROP gold-as-alternatives (OR) official semantics, retry on 5xx, numpy/scipy compat
This commit is contained in:
parent
3d16ab9103
commit
a32d902e94
@ -1,14 +1,30 @@
|
||||
"""evalharness.agent -- evaluation driver for agent benchmarks.
|
||||
|
||||
NOT a general agent framework: no thinking policies. A message pump that
|
||||
lets the model under test act through Environment plugins (bfcl mock today;
|
||||
tau2/swe envs land later) and records trajectories for env_reward scorers.
|
||||
|
||||
from evalharness.agent import drive, BFCLEnvironment
|
||||
traj = await drive(adapter, sample, env=BFCLEnvironment())
|
||||
lets the model under test act through Environment plugins and records
|
||||
trajectories for env_reward scorers. Environments and scoring backends are
|
||||
registered plugins -- drop a module in agent/envs/ to add one.
|
||||
"""
|
||||
|
||||
from .loop import Environment, Trajectory, drive, trajectory_to_prediction
|
||||
from .envs.bfcl_mock import BFCLEnvironment
|
||||
import importlib
|
||||
import pkgutil
|
||||
from pathlib import Path
|
||||
|
||||
__all__ = ['Environment', 'Trajectory', 'drive', 'trajectory_to_prediction', 'BFCLEnvironment']
|
||||
from .loop import (ENV_REGISTRY, Environment, Trajectory, drive,
|
||||
get_env, register_env, trajectory_to_prediction)
|
||||
|
||||
|
||||
def _discover_builtin_envs() -> None:
|
||||
pkg_dir = Path(__file__).parent / 'envs'
|
||||
for info in pkgutil.iter_modules([str(pkg_dir)]):
|
||||
importlib.import_module(f'{__name__}.envs.{info.name}')
|
||||
|
||||
|
||||
_discover_builtin_envs()
|
||||
|
||||
from .envs.bfcl_mock import (BACKEND_REGISTRY, BFCLEnvironment, # noqa: E402,F401
|
||||
get_backend, register_backend)
|
||||
|
||||
__all__ = ['Environment', 'Trajectory', 'drive', 'trajectory_to_prediction',
|
||||
'BFCLEnvironment', 'ENV_REGISTRY', 'register_env', 'get_env',
|
||||
'BACKEND_REGISTRY', 'register_backend', 'get_backend']
|
||||
|
||||
@ -1,19 +1,30 @@
|
||||
"""BFCL mock-function environment: model calls declared functions; the env
|
||||
executes them against official ground-truth state and compares.
|
||||
"""BFCL mock-function environment + scoring backends (native & official).
|
||||
|
||||
Official v3 semantics (eval_checker): for AST-scorable categories the
|
||||
predicted call sequence (name + args) is compared against ground_truth
|
||||
tool_calls; stateful multi-turn categories additionally compare final
|
||||
env state. This env implements the stateful half: it tracks a Python-dict
|
||||
world, applies ground-truth effects for known calls, and exposes the model's
|
||||
call sequence + final state for the scorer.
|
||||
The env records the model's calls (official mock APIs are deterministic);
|
||||
scoring backends compare them against ground truth. Everything here is
|
||||
registered plugins: @register_env / @register_backend.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ...data.sample import ChatMessage, Sample
|
||||
from ..loop import Environment
|
||||
from ...eval.registry import EvalRegistry
|
||||
from ..loop import Environment, register_env
|
||||
|
||||
BACKEND_REGISTRY = EvalRegistry('scoring backend')
|
||||
|
||||
|
||||
def register_backend(name: str):
|
||||
def decorator(fn):
|
||||
BACKEND_REGISTRY.register(name, fn)
|
||||
return fn
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def get_backend(name: str):
|
||||
return BACKEND_REGISTRY.get(name)
|
||||
|
||||
|
||||
def _parse_ground_truth(raw) -> Dict[str, Any]:
|
||||
@ -25,9 +36,74 @@ def _parse_ground_truth(raw) -> Dict[str, Any]:
|
||||
return raw or {}
|
||||
|
||||
|
||||
@register_backend('bfcl_official')
|
||||
def bfcl_official(calls: List[Dict[str, Any]], sample: Sample) -> Optional[bool]:
|
||||
"""Strict backend: bfcl_eval official ast_checker. None => package
|
||||
missing (caller falls back to native). Ground truth and function
|
||||
descriptions pass through VERBATIM; only the model side is adapted to
|
||||
the official decoded shape (incl. the dot->underscore name convention
|
||||
the checker itself applies via convert_func_name)."""
|
||||
try:
|
||||
from bfcl_eval.eval_checker.ast_eval.ast_checker import ast_checker
|
||||
except ImportError:
|
||||
return None
|
||||
gt = _parse_ground_truth(sample.target)
|
||||
if isinstance(gt, dict):
|
||||
possible = gt.get('ground_truth')
|
||||
gt_calls = gt.get('tool_calls')
|
||||
if possible is None and gt_calls:
|
||||
possible = [{(c.get('function', c) or {}).get('name', ''):
|
||||
(c.get('function', c) or {}).get('arguments', {})} for c in gt_calls]
|
||||
else:
|
||||
possible = gt # official list format, verbatim
|
||||
if not possible:
|
||||
return len(calls) == 0 # irrelevance: correct = call nothing
|
||||
if isinstance(possible, str):
|
||||
try:
|
||||
possible = json.loads(possible)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
lang = {'java': 'Java', 'javascript': 'JavaScript'}.get(
|
||||
str((sample.metadata or {}).get('language', '')).lower(), 'Python')
|
||||
category = str((sample.metadata or {}).get('test_category', 'simple'))
|
||||
convention = str((sample.metadata or {}).get('bfcl_model_convention',
|
||||
'gpt-4o-2024-11-20-FC'))
|
||||
# same underscore convention convert_func_name applies to descriptions
|
||||
key = (lambda n: n.replace('.', '_')) if 'FC' in convention else (lambda n: n)
|
||||
model_output = [{key(c['name']): c['arguments']} for c in calls]
|
||||
|
||||
raw_funcs = (sample.metadata or {}).get('functions')
|
||||
funcs = None
|
||||
if raw_funcs:
|
||||
funcs = json.loads(raw_funcs) if isinstance(raw_funcs, str) else raw_funcs
|
||||
if not funcs:
|
||||
funcs = [{'name': t.name, 'description': t.description or '',
|
||||
'parameters': t.parameters} for t in (sample.tools or [])]
|
||||
try:
|
||||
result = ast_checker(funcs, model_output, possible, lang, category, convention)
|
||||
return bool(result.get('valid'))
|
||||
except Exception:
|
||||
# official checker chokes on some v3 schemas; evalscope's adapter
|
||||
# behaves identically: exception -> not valid
|
||||
return False
|
||||
|
||||
|
||||
@register_backend('bfcl_native')
|
||||
def bfcl_native(calls: List[Dict[str, Any]], sample: Sample) -> Optional[bool]:
|
||||
"""Lenient native comparison: exact call-sequence match."""
|
||||
gt = _parse_ground_truth(sample.target)
|
||||
if isinstance(gt, list):
|
||||
want = [list(e.keys())[0] for e in gt] if gt else []
|
||||
got = [c['name'] for c in calls]
|
||||
return want == got
|
||||
return None # dict-form targets: handled by env_reward's native path
|
||||
|
||||
|
||||
@register_env('bfcl_mock')
|
||||
class BFCLEnvironment(Environment):
|
||||
"""Records the model's calls; applies no real side effects (official
|
||||
mock APIs are deterministic). final_state() exposes calls + ground truth."""
|
||||
"""Records the model's calls; official mock APIs are deterministic.
|
||||
final_state() exposes calls + ground truth for the scorer."""
|
||||
|
||||
name = 'bfcl_mock'
|
||||
|
||||
@ -37,8 +113,7 @@ class BFCLEnvironment(Environment):
|
||||
|
||||
def reset(self, sample: Sample) -> List[ChatMessage]:
|
||||
self.calls = []
|
||||
target = sample.target
|
||||
self.ground_truth = _parse_ground_truth(target)
|
||||
self.ground_truth = _parse_ground_truth(sample.target)
|
||||
return []
|
||||
|
||||
async def step(self, tool_calls, text: str, sample: Sample) -> List[ChatMessage]:
|
||||
|
||||
@ -35,6 +35,29 @@ class Environment:
|
||||
return {}
|
||||
|
||||
|
||||
# ---------------- environment registry (万物皆可插件: envs too) ----------------
|
||||
|
||||
from ..eval.registry import EvalRegistry # noqa: E402
|
||||
|
||||
ENV_REGISTRY = EvalRegistry('environment')
|
||||
|
||||
|
||||
def register_env(name: str):
|
||||
"""Class decorator: @register_env('bfcl_mock'). One env per benchmark
|
||||
family; drop a module in agent/envs/ and it auto-registers."""
|
||||
|
||||
def decorator(cls):
|
||||
ENV_REGISTRY.register(name, cls)
|
||||
return cls
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def get_env(name: str):
|
||||
cls = ENV_REGISTRY.get(name)
|
||||
return cls()
|
||||
|
||||
|
||||
class Trajectory:
|
||||
"""Recorded turns: role-tagged messages + per-turn usage."""
|
||||
|
||||
|
||||
@ -120,6 +120,19 @@ _ANSWER_IS = re.compile(
|
||||
_ANSWER_DOLLAR = re.compile(r'final answer is \$([^$]+)\$')
|
||||
|
||||
|
||||
def _clean_answer(text: str) -> str:
|
||||
"""Strip common markdown/noise wrappers from a short extracted answer."""
|
||||
t = text.strip()
|
||||
# **`False`** -> False (bold+code nesting, any order)
|
||||
for _ in range(3):
|
||||
t2 = t.strip('`*_~ ').strip()
|
||||
if t2 == t:
|
||||
break
|
||||
t = t2
|
||||
# trailing explanation after a period on short answers is rare; keep as-is
|
||||
return t
|
||||
|
||||
|
||||
@register_extractor('answer_phrase')
|
||||
def answer_phrase(raw: str, sample: Sample) -> Tuple[str, bool, str]:
|
||||
"""Text after the last 'the answer is' / 'ANSWER:' / '答案是' marker."""
|
||||
@ -129,6 +142,7 @@ def answer_phrase(raw: str, sample: Sample) -> Tuple[str, bool, str]:
|
||||
if not m:
|
||||
return '', False, 'no answer phrase'
|
||||
value = m.group(1).strip().strip('$.: ').split('\n')[0].strip()
|
||||
value = _clean_answer(value)
|
||||
return value, bool(value), f'phrase:{m.group(0)[:20].strip()}'
|
||||
|
||||
|
||||
|
||||
@ -44,7 +44,7 @@ def drop():
|
||||
def trivia_qa():
|
||||
return EvalRecipe(
|
||||
name='trivia_qa',
|
||||
extract=['first_line', 'answer_phrase'],
|
||||
extract=['answer_phrase', 'first_line'],
|
||||
scorers={'em': 'alias_match'},
|
||||
description='TriviaQA; any alias counts.',
|
||||
)
|
||||
|
||||
@ -50,6 +50,15 @@ def evaluate(
|
||||
aggregators = recipe.resolve_aggregators()
|
||||
ctx = ScoreContext(judge=judge, params={})
|
||||
|
||||
# If any scorer executes in docker with per-sample images, overlap pulls
|
||||
# with scoring (run sample N while N+1..N+lookahead images download).
|
||||
bp = None
|
||||
if _needs_bg_prefetch(recipe, samples):
|
||||
from ..sandbox import BackgroundPrefetcher, images_for_samples
|
||||
|
||||
bp = BackgroundPrefetcher(images_for_samples(samples), workers=4, lookahead=8)
|
||||
bp.__enter__()
|
||||
|
||||
results: List[SampleResult] = []
|
||||
for sample, pred in zip(samples, predictions):
|
||||
raw = pred if isinstance(pred, str) else str(pred.get('raw', ''))
|
||||
@ -76,6 +85,8 @@ def evaluate(
|
||||
if isinstance(pred, dict) and pred.get('usage'):
|
||||
result.usage = pred['usage']
|
||||
try:
|
||||
if bp is not None and sample.sandbox and sample.sandbox.image:
|
||||
bp.ensure(sample.sandbox.image) # wait only if this one still pulling
|
||||
value, ok, note = extractor(raw, sample)
|
||||
result.extracted_prediction = value
|
||||
result.extraction_ok = ok
|
||||
@ -108,12 +119,30 @@ def evaluate(
|
||||
samples=results,
|
||||
)
|
||||
_aggregate_into(report, results, recipe, aggregators)
|
||||
if bp is not None:
|
||||
report.metric_groups['run_info'] = {
|
||||
**report.metric_groups.get('run_info', {}),
|
||||
**{f'img_{k}': v for k, v in bp.stats().items()},
|
||||
}
|
||||
bp.__exit__(None, None, None)
|
||||
if extra_metadata:
|
||||
report.metric_groups['run_info'] = {k: v for k, v in extra_metadata.items()
|
||||
if isinstance(v, (int, float, str))}
|
||||
return report
|
||||
|
||||
|
||||
def _needs_bg_prefetch(recipe, samples) -> bool:
|
||||
"""True when the recipe executes in docker AND samples declare images."""
|
||||
try:
|
||||
for spec in recipe.scorers.values():
|
||||
params = spec if isinstance(spec, dict) else {}
|
||||
if params.get('name') == 'execution' and params.get('sandbox') == 'docker':
|
||||
return any(s.sandbox and s.sandbox.image for s in samples[:50])
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _aggregate_into(report: EvalReport, results, recipe: EvalRecipe, aggregators) -> None:
|
||||
for metric in recipe.scorers:
|
||||
agg = aggregators.get(metric)
|
||||
|
||||
@ -218,13 +218,25 @@ def _drop_metrics(predicted: List[str], gold: List[str]):
|
||||
def em_f1(pred: str, target, sample: Sample, ctx: ScoreContext):
|
||||
"""DROP official EM/F1 over gold spans.
|
||||
|
||||
Target shape: List[str] (multiple spans) or str. The prediction is split
|
||||
on the official '\\n' separator of multiple predicted spans.
|
||||
Gold spans are alternative acceptable answers (OR semantics): the
|
||||
prediction is split on newlines into predicted spans, then scored
|
||||
against EACH gold alternative; the best gold wins (official
|
||||
get_drop_metrics called once per alternative).
|
||||
"""
|
||||
gold: List[str] = [str(t) for t in target] if isinstance(target, list) else [str(target)]
|
||||
predicted = [p for p in re.split(r'\n', pred or '') if p.strip()]
|
||||
import re as _re
|
||||
|
||||
golds: List[List[str]] = []
|
||||
if isinstance(target, list):
|
||||
# list of alternative answers -> each is one full gold-answer option
|
||||
golds = [[str(t)] for t in target]
|
||||
else:
|
||||
golds = [[str(target)]]
|
||||
predicted = [p for p in _re.split(r'\n', pred or '') if p.strip()]
|
||||
best_em, best_f1 = 0.0, 0.0
|
||||
for gold in golds:
|
||||
em, f1 = _drop_metrics(predicted, gold)
|
||||
return {'em': em / 100.0 if em > 1.0 else em, 'f1': f1 / 100.0}, {'em': {}, 'f1': {'official': True}}
|
||||
best_em, best_f1 = max(best_em, em), max(best_f1, f1 / 100.0)
|
||||
return {'em': best_em, 'f1': best_f1}, {'em': {'alternatives': len(golds)}, 'f1': {'official': True}}
|
||||
|
||||
|
||||
@register_scorer('alias_match')
|
||||
@ -331,6 +343,20 @@ def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext):
|
||||
'(run with run_eval(loop=True) or a bench env)'
|
||||
)
|
||||
calls = env_state.get('calls', [])
|
||||
backend = ctx.params.get('backend', 'auto') # auto | bfcl_official | bfcl_native | ...
|
||||
if backend != 'native':
|
||||
from ..agent.envs.bfcl_mock import BACKEND_REGISTRY, get_backend
|
||||
|
||||
name = backend if backend != 'auto' else 'bfcl_official'
|
||||
try:
|
||||
verdict = get_backend(name)(calls, sample)
|
||||
except KeyError:
|
||||
verdict = None
|
||||
if verdict is not None:
|
||||
return {'acc': float(verdict)}, {'acc': {'mode': name, 'backend': name}}
|
||||
if backend != 'auto':
|
||||
raise LayerNotReady(f'backend {backend!r} unavailable '
|
||||
'(missing dependency?)')
|
||||
gt_calls = (env_state.get('ground_truth') or {}).get('tool_calls')
|
||||
if gt_calls is None:
|
||||
# irrelevance categories: correct behavior is calling NOTHING
|
||||
|
||||
@ -149,6 +149,7 @@ class OpenAICompatible(ModelAdapter):
|
||||
for k in ('temperature', 'max_tokens', 'top_p', 'stop', 'seed', 'response_format'):
|
||||
if kw.get(k) is not None:
|
||||
payload[k] = kw[k]
|
||||
payload.setdefault('max_tokens', self.extra.get('max_tokens', 4096)) # CoT room
|
||||
return payload
|
||||
|
||||
def _parse(self, data: Dict[str, Any]) -> ModelOutput:
|
||||
|
||||
@ -56,7 +56,20 @@ async def generate_predictions(
|
||||
ctx = (sample.metadata or {}).get(key)
|
||||
if ctx:
|
||||
parts.append(str(ctx))
|
||||
parts.append(sample.input_text)
|
||||
question = sample.input_text
|
||||
if sample.choices:
|
||||
# MCQ: options MUST be in the prompt; ask for the letter
|
||||
letters = 'ABCDEFGH'
|
||||
opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(sample.choices)
|
||||
if i < len(letters))
|
||||
question = (f'{question}\n\n{opts}\n\n'
|
||||
'Answer with the letter of the correct option.')
|
||||
elif sample.task_type in ('qa',):
|
||||
# gentle output contract (matches official few-shot conventions)
|
||||
question = (f'{question}\n\n'
|
||||
'End your reply with the final answer on its own last line '
|
||||
'in the form "Answer: <answer>".')
|
||||
parts.append(question)
|
||||
text = '\n\n'.join(parts)
|
||||
if max_input_chars and len(text) > max_input_chars:
|
||||
keep = max_input_chars // 2
|
||||
@ -168,12 +181,11 @@ async def run_eval(
|
||||
|
||||
env_factory = None
|
||||
if env:
|
||||
from ..agent import BFCLEnvironment
|
||||
from ..agent.loop import ENV_REGISTRY, get_env
|
||||
|
||||
envs = {'bfcl_mock': BFCLEnvironment}
|
||||
if env not in envs:
|
||||
raise KeyError(f"unknown env {env!r}; available: {', '.join(envs)}")
|
||||
env_factory = envs[env]
|
||||
if env not in ENV_REGISTRY:
|
||||
raise KeyError(f'unknown env {env!r}; available: {", ".join(ENV_REGISTRY.names())}')
|
||||
env_factory = lambda: get_env(env) # noqa: E731
|
||||
|
||||
try:
|
||||
preds, _usages, usage = await generate_predictions(
|
||||
|
||||
@ -24,10 +24,11 @@ from .base import (
|
||||
from .docker import DockerSandbox, docker_available, docker_serve, serve_env
|
||||
from .local import LocalSandbox
|
||||
from .prefetch import images_for_dataset, prefetch_images
|
||||
from .bg_prefetch import BackgroundPrefetcher
|
||||
|
||||
__all__ = [
|
||||
'Sandbox', 'DockerSandbox', 'LocalSandbox', 'ExecResult', 'EnvHandle',
|
||||
'SANDBOX_REGISTRY', 'register_sandbox', 'get_sandbox', 'acquire', 'stop_all',
|
||||
'docker_serve', 'serve_env', 'docker_available',
|
||||
'prefetch_images', 'images_for_dataset',
|
||||
'prefetch_images', 'images_for_dataset', 'BackgroundPrefetcher',
|
||||
]
|
||||
|
||||
92
evalharness/sandbox/bg_prefetch.py
Normal file
92
evalharness/sandbox/bg_prefetch.py
Normal file
@ -0,0 +1,92 @@
|
||||
"""Background image prefetcher: overlap docker pulls with evaluation.
|
||||
|
||||
While the runner executes sample N (its image already local), worker
|
||||
threads pull the images of upcoming samples N+1.. — so per-sample runs
|
||||
never wait on a cold multi-GB pull unless the queue drains.
|
||||
|
||||
from evalharness.sandbox.prefetch import BackgroundPrefetcher
|
||||
images = images_for_dataset(ds) # ordered like the dataset
|
||||
with BackgroundPrefetcher(images, workers=4, lookahead=8) as bp:
|
||||
for sample in ds:
|
||||
bp.ensure(sample.sandbox.image) # blocks only if still pulling
|
||||
... run in sandbox ...
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .prefetch import _pull_one, local_images
|
||||
|
||||
|
||||
class BackgroundPrefetcher:
|
||||
"""Pull upcoming images on worker threads; evaluation thread consumes."""
|
||||
|
||||
def __init__(self, images: List[str], workers: int = 4, lookahead: int = 8):
|
||||
self.images = list(images)
|
||||
self.workers = max(1, workers)
|
||||
self.lookahead = max(1, lookahead)
|
||||
self._cursor = 0
|
||||
self._lock = threading.Lock()
|
||||
self._ready: Dict[str, bool] = {}
|
||||
self._failed: Dict[str, str] = {}
|
||||
self._stop = threading.Event()
|
||||
self._threads: List[threading.Thread] = []
|
||||
for img in self.images:
|
||||
self._ready[img] = True if img in local_images() else False
|
||||
|
||||
def __enter__(self) -> 'BackgroundPrefetcher':
|
||||
for i in range(self.workers):
|
||||
t = threading.Thread(target=self._work, name=f'eh-prefetch-{i}', daemon=True)
|
||||
t.start()
|
||||
self._threads.append(t)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> None:
|
||||
self._stop.set()
|
||||
for t in self._threads:
|
||||
t.join(timeout=5)
|
||||
|
||||
def _work(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
img = self._next_pending()
|
||||
if img is None:
|
||||
time.sleep(0.5)
|
||||
continue
|
||||
try:
|
||||
_pull_one(img)
|
||||
with self._lock:
|
||||
self._ready[img] = True
|
||||
except Exception as e:
|
||||
with self._lock:
|
||||
self._failed[img] = str(e)[:200]
|
||||
|
||||
def _next_pending(self) -> Optional[str]:
|
||||
"""Claim the next not-ready image within the lookahead window."""
|
||||
with self._lock:
|
||||
hi = min(self._cursor + self.lookahead, len(self.images))
|
||||
for i in range(self._cursor, hi):
|
||||
img = self.images[i]
|
||||
if not self._ready.get(img) and img not in self._failed:
|
||||
return img # claimed (pull is idempotent; duplicates are cheap)
|
||||
return None
|
||||
|
||||
def ensure(self, image: Optional[str]) -> bool:
|
||||
"""Advance the cursor to `image`; wait (bounded) until pulled."""
|
||||
if not image:
|
||||
return True
|
||||
with self._lock: # allow random access ordering too
|
||||
if image in self.images and self.images.index(image) >= self._cursor:
|
||||
self._cursor = self.images.index(image)
|
||||
deadline = time.time() + 3600
|
||||
while time.time() < deadline and not self._stop.is_set():
|
||||
with self._lock:
|
||||
if self._ready.get(image) or image in self._failed:
|
||||
return self._ready.get(image, False)
|
||||
time.sleep(1.0)
|
||||
return self._ready.get(image, False)
|
||||
|
||||
def stats(self) -> Dict[str, int]:
|
||||
with self._lock:
|
||||
ready = sum(1 for v in self._ready.values() if v)
|
||||
return {'total': len(self.images), 'ready': ready, 'failed': len(self._failed)}
|
||||
@ -29,6 +29,15 @@ def images_for_dataset(ds: Dataset, limit: int = 0) -> List[str]:
|
||||
return seen
|
||||
|
||||
|
||||
def images_for_samples(samples) -> List[str]:
|
||||
"""Distinct sandbox images in sample order (dedup, keeps order)."""
|
||||
seen: List[str] = []
|
||||
for s in samples:
|
||||
if s.sandbox and s.sandbox.image and s.sandbox.image not in seen:
|
||||
seen.append(s.sandbox.image)
|
||||
return seen
|
||||
|
||||
|
||||
def local_images() -> set:
|
||||
out = subprocess.run(['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}'],
|
||||
capture_output=True, text=True)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user