13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
36 lines
1.5 KiB
Diff
36 lines
1.5 KiB
Diff
--- /tmp/evalscope-sync/src-v191/evalscope/benchmarks/swe_bench/utils.py 2026-07-21 03:18:18.000000000 +0000
|
|
+++ /data1/syy/evalscope/evalstone/evalscope/evalscope/benchmarks/swe_bench/utils.py 2026-08-21 03:03:19.617324068 +0000
|
|
@@ -83,6 +83,8 @@
|
|
|
|
# Build + start instance container (instance image should already be built)
|
|
container = None
|
|
+ client = None
|
|
+ instance_id = None
|
|
eval_completed = False
|
|
report = {}
|
|
try:
|
|
@@ -176,15 +178,20 @@
|
|
f"Result for {instance_id}: resolved: {report[instance_id]['resolved']}")
|
|
eval_completed = True
|
|
except Exception as e:
|
|
- error_msg = (f'Error in evaluating model for {instance_id}: {e}\n'
|
|
+ display_id = instance_id or 'unknown'
|
|
+ error_msg = (f'Error in evaluating model for {display_id}: {e}\n'
|
|
f'{traceback.format_exc()}')
|
|
logger.error(error_msg)
|
|
report['error'] = error_msg
|
|
finally:
|
|
# Remove instance container
|
|
- cleanup_container(client, container, logger)
|
|
+ if client is not None:
|
|
+ cleanup_container(client, container, logger)
|
|
+ resolved = False
|
|
+ if instance_id is not None:
|
|
+ resolved = report.get(instance_id, {}).get('resolved', False)
|
|
return {
|
|
'completed': eval_completed,
|
|
- 'resolved': report.get(instance_id, {}).get('resolved', False),
|
|
+ 'resolved': resolved,
|
|
'report': report,
|
|
}
|