227ef603a6
- Restructure repo around experiments/<name>/ and platforms/<chip>.env. - Add shared scripts under scripts/common/ for platform/server/bench-client logic. - Add Kunlun P800 platform config and runtime patches. - Add dsv4_p800_sglang experiment with INT8 smoke-test support. - Update BENCHMARK_WORKFLOW.md and README.md with chip/engine recording rules. - Add scripts/analysis/compare_experiments.py for cross-experiment comparison. - Ignore experiments/*/results/ raw output directories by default.
154 lines
3.8 KiB
Bash
Executable File
154 lines
3.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Common helpers for benchmark orchestrators.
|
|
# Usage: source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/../scripts/common/lib.sh"
|
|
|
|
set -Eeuo pipefail
|
|
|
|
# Resolve repository root relative to this file.
|
|
_COMMON_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT_DIR="$(cd "${_COMMON_LIB_DIR}/../.." && pwd)"
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Logging
|
|
# -----------------------------------------------------------------------------
|
|
|
|
LOG_FILE=""
|
|
|
|
log_init() {
|
|
local log_path="$1"
|
|
LOG_FILE="$log_path"
|
|
mkdir -p "$(dirname "$LOG_FILE")"
|
|
: > "$LOG_FILE"
|
|
}
|
|
|
|
log() {
|
|
local msg="[$(date --iso-8601=seconds)] $*"
|
|
echo "$msg"
|
|
if [[ -n "${LOG_FILE:-}" ]]; then
|
|
echo "$msg" >> "$LOG_FILE"
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Result directories
|
|
# -----------------------------------------------------------------------------
|
|
|
|
ensure_result_root() {
|
|
local result_root="$1"
|
|
mkdir -p "${result_root}/logs"
|
|
mkdir -p "${result_root}/raw_outputs"
|
|
echo "$result_root"
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Server health check
|
|
# -----------------------------------------------------------------------------
|
|
|
|
health_check() {
|
|
local host="${1:-127.0.0.1}"
|
|
local port="${2:-30000}"
|
|
local max_wait="${3:-120}"
|
|
|
|
for ((i = 1; i <= max_wait; i++)); do
|
|
if curl --fail --silent --show-error --max-time 5 "http://${host}:${port}/health" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Git metadata
|
|
# -----------------------------------------------------------------------------
|
|
|
|
git_commit() {
|
|
cd "$ROOT_DIR" || return 1
|
|
git rev-parse --short HEAD 2>/dev/null || echo "unknown"
|
|
}
|
|
|
|
git_dirty() {
|
|
cd "$ROOT_DIR" || return 1
|
|
if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then
|
|
echo "dirty"
|
|
else
|
|
echo "clean"
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# results.json metadata
|
|
# -----------------------------------------------------------------------------
|
|
|
|
write_metadata_json() {
|
|
local output_path="$1"
|
|
local experiment="$2"
|
|
local run_id="$3"
|
|
local model="$4"
|
|
local backend="$5"
|
|
local engine="$6"
|
|
local hardware="$7"
|
|
local accelerator="$8"
|
|
local chip="$9"
|
|
local script="${10}"
|
|
local env_path="${11:-}"
|
|
local description="${12:-}"
|
|
|
|
mkdir -p "$(dirname "$output_path")"
|
|
|
|
cat > "$output_path" <<EOF
|
|
{
|
|
"metadata": {
|
|
"experiment": "${experiment}",
|
|
"run_id": "${run_id}",
|
|
"timestamp": "$(date --iso-8601=seconds)",
|
|
"model": "${model}",
|
|
"backend": "${backend}",
|
|
"engine": "${engine}",
|
|
"hardware": "${hardware}",
|
|
"accelerator": "${accelerator}",
|
|
"chip": "${chip}",
|
|
"script": "${script}",
|
|
"env": "${env_path}",
|
|
"git_commit": "$(git_commit)",
|
|
"git_dirty": "$(git_dirty)",
|
|
"description": "${description}"
|
|
},
|
|
"config": {},
|
|
"scenarios": []
|
|
}
|
|
EOF
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# JSON append helper (naive but sufficient for small result JSONs)
|
|
# -----------------------------------------------------------------------------
|
|
|
|
append_scenario_to_json() {
|
|
local json_path="$1"
|
|
local scenario_json="$2"
|
|
|
|
if [[ ! -f "$json_path" ]]; then
|
|
echo "ERROR: $json_path not found" >&2
|
|
return 1
|
|
fi
|
|
|
|
# Use Python for safe JSON manipulation.
|
|
"${PYTHON:-python3}" - "$json_path" "$scenario_json" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
json_path = sys.argv[1]
|
|
scenario_json = sys.argv[2]
|
|
|
|
with open(json_path, "r", encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
|
|
scenario = json.loads(scenario_json)
|
|
data["scenarios"].append(scenario)
|
|
|
|
with open(json_path, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
PY
|
|
}
|