64 lines
2.0 KiB
Python
64 lines
2.0 KiB
Python
"""Visualization layer: consume EvalReport artifacts, render views.
|
|
|
|
Strictly a CONSUMER of the eval layer (reads saved report json, never scores).
|
|
Renderers are registered plugins: text table, markdown, and ascii bar/radar
|
|
charts today; a web renderer can register the same way later.
|
|
|
|
from evalharness.viz import render
|
|
render('report.json', style='text') # console table
|
|
render([r1, r2], style='md_compare') # benchmark comparison table
|
|
"""
|
|
|
|
import importlib
|
|
import pkgutil
|
|
from pathlib import Path
|
|
from typing import Callable, Dict, List, Union
|
|
|
|
from ..eval.record import EvalReport
|
|
|
|
RendererFn = Callable[[Union['EvalReport', List['EvalReport'], str, Path], Dict], str]
|
|
|
|
RENDERERS: Dict[str, RendererFn] = {}
|
|
|
|
|
|
def register_renderer(name: str):
|
|
def decorator(fn: RendererFn) -> RendererFn:
|
|
if name in RENDERERS:
|
|
raise ValueError(f'renderer {name!r} already registered')
|
|
RENDERERS[name] = fn
|
|
return fn
|
|
|
|
return decorator
|
|
|
|
|
|
def get_renderer(name: str) -> RendererFn:
|
|
if name not in RENDERERS:
|
|
raise KeyError(f"unknown renderer {name!r}. Available: {', '.join(sorted(RENDERERS))}")
|
|
return RENDERERS[name]
|
|
|
|
|
|
def render(target, style: str = 'text', **opts) -> str:
|
|
"""Render one report path/object or a list of them (comparison styles)."""
|
|
return get_renderer(style)(_load(target), opts)
|
|
|
|
|
|
def _load(target) -> Union[EvalReport, List[EvalReport]]:
|
|
if isinstance(target, (str, Path)):
|
|
return EvalReport.load(target)
|
|
if isinstance(target, EvalReport):
|
|
return target
|
|
if isinstance(target, (list, tuple)):
|
|
return [_load(t) for t in target]
|
|
raise TypeError(f'cannot load report from {type(target)}')
|
|
|
|
|
|
def _discover_builtin_renderers() -> None:
|
|
pkg_dir = Path(__file__).parent / 'renderers'
|
|
if not pkg_dir.exists():
|
|
return
|
|
for info in pkgutil.iter_modules([str(pkg_dir)]):
|
|
importlib.import_module(f'{__name__}.renderers.{info.name}')
|
|
|
|
|
|
_discover_builtin_renderers()
|