13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
97 lines
4.0 KiB
Python
97 lines
4.0 KiB
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
"""Unit tests for the eval-report delete endpoint (DELETE /api/v1/reports/runs/<run_id>/models/<model_id>).
|
|
|
|
Builds a fake eval output tree with two model reports sharing one run
|
|
directory and verifies per-model deletion, run-dir cleanup, path-traversal
|
|
rejection and running-task protection.
|
|
|
|
Skipped automatically when Flask (service extra) is not installed.
|
|
"""
|
|
import json
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
import pytest
|
|
|
|
flask = pytest.importorskip('flask') # noqa: F841 (service extra not installed → skip)
|
|
|
|
|
|
def _write_json(path: str, obj: object) -> None:
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, 'w', encoding='utf-8') as f:
|
|
json.dump(obj, f)
|
|
|
|
|
|
class TestReportDelete(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
from evalscope.service.app import create_app
|
|
|
|
self.tmp = tempfile.mkdtemp()
|
|
self.prefix = '20260101_120000'
|
|
run_dir = os.path.join(self.tmp, self.prefix)
|
|
# Two models share the run directory; each has reports + predictions.
|
|
for model in ('model-a', 'model-b'):
|
|
_write_json(os.path.join(run_dir, 'reports', model, 'gsm8k.json'), {})
|
|
_write_json(os.path.join(run_dir, 'predictions', model, 'gsm8k.jsonl'), {})
|
|
with open(os.path.join(run_dir, 'reports', 'report.html'), 'w', encoding='utf-8') as f:
|
|
f.write('<html><body>model-a model-b</body></html>')
|
|
os.makedirs(os.path.join(run_dir, 'logs'), exist_ok=True)
|
|
os.makedirs(os.path.join(run_dir, 'configs'), exist_ok=True)
|
|
|
|
self.client = create_app().test_client()
|
|
|
|
def tearDown(self):
|
|
shutil.rmtree(self.tmp, ignore_errors=True)
|
|
|
|
def _delete(self, run_id: str, model_id: str):
|
|
return self.client.delete(
|
|
f'/api/v1/reports/runs/{run_id}/models/{model_id}',
|
|
query_string={'root_path': self.tmp},
|
|
)
|
|
|
|
def test_delete_one_model_keeps_run_dir(self):
|
|
res = self._delete(self.prefix, 'model-a')
|
|
self.assertEqual(res.status_code, 200)
|
|
body = res.get_json()
|
|
self.assertTrue(body['success'])
|
|
run_dir = os.path.join(self.tmp, self.prefix)
|
|
# model-a artefacts are gone; model-b and the run dir survive.
|
|
self.assertFalse(os.path.isdir(os.path.join(run_dir, 'reports', 'model-a')))
|
|
self.assertFalse(os.path.isdir(os.path.join(run_dir, 'predictions', 'model-a')))
|
|
self.assertTrue(os.path.isdir(os.path.join(run_dir, 'reports', 'model-b')))
|
|
self.assertTrue(os.path.isdir(run_dir))
|
|
html_report = os.path.join(run_dir, 'reports', 'report.html')
|
|
if os.path.isfile(html_report):
|
|
with open(html_report, 'r', encoding='utf-8') as f:
|
|
self.assertNotIn('model-a', f.read())
|
|
|
|
def test_delete_last_model_removes_run_dir(self):
|
|
self._delete(self.prefix, 'model-a')
|
|
res = self._delete(self.prefix, 'model-b')
|
|
self.assertEqual(res.status_code, 200)
|
|
self.assertFalse(os.path.isdir(os.path.join(self.tmp, self.prefix)))
|
|
# The outputs root itself is never removed.
|
|
self.assertTrue(os.path.isdir(self.tmp))
|
|
|
|
def test_delete_rejects_bad_names(self):
|
|
# Unknown model report under an existing run.
|
|
self.assertEqual(self._delete(self.prefix, 'ghost').status_code, 404)
|
|
# Unknown run directory.
|
|
self.assertEqual(self._delete('20990101_000000', 'model-a').status_code, 404)
|
|
# The tree is untouched by all rejected attempts.
|
|
self.assertTrue(os.path.isdir(os.path.join(self.tmp, self.prefix, 'reports', 'model-a')))
|
|
|
|
def test_delete_rejects_running_task(self):
|
|
with mock.patch('evalscope.service.blueprints.reports.active_task_ids', return_value={self.prefix}):
|
|
res = self._delete(self.prefix, 'model-a')
|
|
self.assertEqual(res.status_code, 409)
|
|
self.assertTrue(os.path.isdir(os.path.join(self.tmp, self.prefix, 'reports', 'model-a')))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|