diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/config.env b/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/config.env new file mode 100644 index 0000000..b8c2844 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/config.env @@ -0,0 +1,67 @@ +# ============================================================ +# DeepSeek-V4-Pro SGLang TP=16 EP=2 + EAGLE speculative decoding +# Nodes: 10.101.0.11 (rank 0) + 10.101.0.13 (rank 1) +# GPUs: 16× RTX 6000D (85GB each) +# +# Key flags: +# SGLANG_SHARED_EXPERT_TP1=1 + --ep-size 2 → TP=16 with FP8 block=128 +# --speculative-algorithm EAGLE → Eagle speculative decoding +# ============================================================ + +EXPERIMENT="dsv4_pro6000_sglang_tp16_eagle" +MODEL_NAME="DeepSeek-V4-Pro" +MODEL_PATH="/data/hf_models/DeepSeek-V4-Pro" +SERVED_MODEL_NAME="default" + +# ---- Nodes (IPs for reliable SSH) ---- +HEAD_NODE="10.101.0.11" +WORKER_NODE="10.101.0.13" +HEAD_IP="10.101.0.11" +DIST_INIT_PORT="${DIST_INIT_PORT:-20000}" +SGLANG_PORT="${SGLANG_PORT:-30000}" + +# ---- Parallelism ---- +TP_SIZE="${TP_SIZE:-16}" +EP_SIZE="${EP_SIZE:-2}" +NNODES="${NNODES:-2}" + +# ---- Docker ---- +DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}" + +# ---- NCCL ---- +NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-eth0}" +NCCL_DEBUG="${NCCL_DEBUG:-WARN}" + +# ---- SGLang memory / perf ---- +# max_running_requests must >= max concurrency tested (128) to avoid queueing +# cuda_graph_max_bs_decode must >= max concurrent batch size for accurate decode perf +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}" +CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-64}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-256}" + +# ---- EAGLE speculative decoding ---- +SPECULATIVE_ALGORITHM="${SPECULATIVE_ALGORITHM:-EAGLE}" +SPECULATIVE_NUM_STEPS="${SPECULATIVE_NUM_STEPS:-3}" +SPECULATIVE_EAGLE_TOPK="${SPECULATIVE_EAGLE_TOPK:-1}" +SPECULATIVE_NUM_DRAFT_TOKENS="${SPECULATIVE_NUM_DRAFT_TOKENS:-4}" + +# ---- Benchmark ---- +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}" +SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}" +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +WARMUP_REQUESTS="${WARMUP_REQUESTS:-16}" +NUM_REPETITIONS="${NUM_REPETITIONS:-3}" + +# ---- Timeouts ---- +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +HEALTH_CHECK_RETRIES="${HEALTH_CHECK_RETRIES:-600}" +HEALTH_CHECK_INTERVAL_S="${HEALTH_CHECK_INTERVAL_S:-5}" + +# ---- Directories ---- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/results}" + +# ---- Dry-run ---- +DRY_RUN="${DRY_RUN:-0}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/run_batch.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/run_batch.sh new file mode 100755 index 0000000..4868c6a --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/run_batch.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +# ================================================================ +# ~20h benchmark matching dsv4_pro6000_vllm_tp_dp_matrix matrix.json +# Per-ISL TTFT SLO: 7s(min) -> 40s(max), stops concurrency sweep on violation +# OOM fault tolerance: auto-detect, skip combo, auto-restart server +# 3 repetitions for statistical robustness +# ================================================================ + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date +%Y%m%d_%H%M%S)}" +RESULT_DIR="${RESULT_BASE}/${RUN_ID}" +SUMMARY_FILE="${RESULT_DIR}/summary.csv" +mkdir -p "$RESULT_DIR" +echo "isl,osl,concurrency,tps,ttft_ms,tpot_ms,e2e_ms,slo_ms,status,elapsed_s" >> "$SUMMARY_FILE" + +log() { echo "[$(date '+%H:%M:%S')] $*"; } + +# ================================================================ +# TTFT SLO per ISL (DeepSeek-V4-Pro, TP=16 cross-node, min=7000ms) +# ================================================================ +get_slo_ms() { + local isl="$1" + if (( isl <= 2048 )); then echo 7000 + elif (( isl <= 8192 )); then echo 10000 + elif (( isl <= 32768 )); then echo 15000 + elif (( isl <= 131072 )); then echo 25000 + else echo 40000 + fi +} + +# ================================================================ +# Reference matrix from dsv4_pro6000_vllm_tp_dp_matrix/matrix.json +# ================================================================ +declare -A MATRIX +MATRIX["1024,128"]=Y; MATRIX["1024,256"]=Y; MATRIX["1024,512"]=Y +MATRIX["1024,1024"]=Y; MATRIX["1024,2048"]=Y; MATRIX["1024,4096"]=Y +MATRIX["4096,128"]=Y; MATRIX["4096,256"]=Y; MATRIX["4096,512"]=Y +MATRIX["4096,1024"]=Y; MATRIX["4096,2048"]=Y; MATRIX["4096,4096"]=Y +MATRIX["8192,128"]=Y; MATRIX["8192,256"]=Y; MATRIX["8192,512"]=Y +MATRIX["8192,1024"]=Y; MATRIX["8192,2048"]=Y; MATRIX["8192,4096"]=Y +MATRIX["16384,128"]=Y; MATRIX["16384,256"]=Y; MATRIX["16384,512"]=Y +MATRIX["16384,1024"]=Y; MATRIX["16384,2048"]=Y; MATRIX["16384,4096"]=P +MATRIX["32768,128"]=Y; MATRIX["32768,256"]=Y; MATRIX["32768,512"]=Y +MATRIX["32768,1024"]=Y; MATRIX["32768,2048"]=Y; MATRIX["32768,4096"]=P +MATRIX["65536,128"]=Y; MATRIX["65536,256"]=Y; MATRIX["65536,512"]=Y +MATRIX["65536,1024"]=Y; MATRIX["65536,2048"]=P; MATRIX["65536,4096"]=N +MATRIX["131072,128"]=Y; MATRIX["131072,256"]=Y; MATRIX["131072,512"]=Y +MATRIX["131072,1024"]=P; MATRIX["131072,2048"]=N; MATRIX["131072,4096"]=N +MATRIX["262144,128"]=Y; MATRIX["262144,256"]=Y; MATRIX["262144,512"]=P +MATRIX["262144,1024"]=N; MATRIX["262144,2048"]=N; MATRIX["262144,4096"]=N +MATRIX["524288,128"]=Y; MATRIX["524288,256"]=P; MATRIX["524288,512"]=N +MATRIX["524288,1024"]=N; MATRIX["524288,2048"]=N; MATRIX["524288,4096"]=N +MATRIX["1048576,128"]=Y; MATRIX["1048576,256"]=P; MATRIX["1048576,512"]=N +MATRIX["1048576,1024"]=N; MATRIX["1048576,2048"]=N; MATRIX["1048576,4096"]=N + +ISL_LIST=(1024 4096 8192 16384 32768 65536 131072 262144 524288 1048576) +OSL_LIST=(128 256 512 1024 2048 4096) + +get_conc_for() { + local isl="$1" + if (( isl <= 4096 )); then echo "1 2 4 8 16 32 64 128" + elif (( isl <= 16384 )); then echo "1 2 4 8 16 32" + elif (( isl <= 65536 )); then echo "1 2 4 8" + else echo "1 2 4" + fi +} + +# ================================================================ +# Benchmark runner with per-ISL SLO + OOM tolerance +# ================================================================ +run_one() { + local isl="$1" osl="$2" conc="$3" + local slo_ms; slo_ms=$(get_slo_ms "$isl") + local num_prompts=$(( conc * 5 )) + [[ $num_prompts -lt 10 ]] && num_prompts=10 + local out="${RESULT_DIR}/isl${isl}_osl${osl}_c${conc}.json" + local logf="${RESULT_DIR}/isl${isl}_osl${osl}_c${conc}.log" + local start_ts=$(date +%s) + + docker run --rm --network host \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${DATASET_PATH}:${DATASET_PATH}:ro" \ + -v "${RESULT_DIR}:${RESULT_DIR}" \ + -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 \ + --entrypoint python3 "$DOCKER_CLIENT_IMAGE" \ + -m "${SGLANG_BENCH_MODULE}" \ + --backend sglang --host "${HEAD_IP}" --port "${SGLANG_PORT}" \ + --dataset-name "${BENCH_DATASET_NAME}" --dataset-path "${DATASET_PATH}" \ + --random-input-len "$isl" --random-output-len "$osl" \ + --random-range-ratio "${RANDOM_RANGE_RATIO}" \ + --num-prompts "$num_prompts" --max-concurrency "$conc" \ + --request-rate 10000 --warmup-requests "${WARMUP_REQUESTS}" \ + --output-file "$out" --output-details --disable-tqdm \ + > "$logf" 2>&1 + local rc=$? + local elapsed=$(($(date +%s) - start_ts)) + + # OOM detection + if [[ $rc -ne 0 ]]; then + if grep -qi "out of memory\|CUDA error\|RESOURCE_EXHAUSTED" "$logf" 2>/dev/null; then + echo "${isl},${osl},${conc},-1,-1,-1,-1,${slo_ms},OOM,${elapsed}" >> "$SUMMARY_FILE" + log " OOM isl=${isl} osl=${osl} c=${conc}"; return 2 + fi + echo "${isl},${osl},${conc},-1,-1,-1,-1,${slo_ms},FAIL,${elapsed}" >> "$SUMMARY_FILE" + log " FAIL isl=${isl} osl=${osl} c=${conc} rc=${rc}"; return 1 + fi + + # Extract metrics (JSONL: first line has aggregate summary) + local metrics=$(python3 -c " +import json +for line in open('${out}'): + line=line.strip() + if not line: continue + try: + d=json.loads(line) + tps=d.get('total_throughput',d.get('tps',-1)) + ttft=d.get('mean_ttft_ms',d.get('ttft_mean',-1)) + tpot=d.get('mean_tpot_ms',d.get('tpot_mean',-1)) + e2e=d.get('mean_e2e_latency_ms',d.get('e2e_latency_mean',-1)) + if tps!=-1 or ttft!=-1: + print(f'{round(tps,1)},{round(ttft,1)},{round(tpot,1)},{round(e2e,1)}') + break + except: pass +" 2>/dev/null || echo "-1,-1,-1,-1") + local tps=$(echo "$metrics" | cut -d, -f1) + local ttft=$(echo "$metrics" | cut -d, -f2) + local tpot=$(echo "$metrics" | cut -d, -f3) + local e2e=$(echo "$metrics" | cut -d, -f4) + + echo "${isl},${osl},${conc},${tps},${ttft},${tpot},${e2e},${slo_ms},OK,${elapsed}" >> "$SUMMARY_FILE" + + # TTFT SLO check (per-ISL) + if [[ "$ttft" != "-1" ]]; then + if (( $(echo "$ttft > ${slo_ms}" | bc -l 2>/dev/null) )); then + log " SLO! ttft=${ttft}ms > slo=${slo_ms}ms -> stop higher conc" + return 3 + fi + fi + + log " OK isl=${isl} osl=${osl} c=${conc} tps=${tps} ttft=${ttft}ms slo=${slo_ms}ms" + return 0 +} + +# ================================================================ +# OOM recovery: restart server if it died +# ================================================================ +check_and_recover_server() { + if curl --fail --silent --max-time 5 "http://${HEAD_IP}:${SGLANG_PORT}/health" >/dev/null 2>&1; then + return 0 + fi + log "WARN: Server health failed. Checking..." + local hok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HEAD_NODE}" \ + "docker ps --filter name=${EXPERIMENT}_head --format '{{.Names}}' 2>/dev/null" || echo "") + local wok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${WORKER_NODE}" \ + "docker ps --filter name=${EXPERIMENT}_worker --format '{{.Names}}' 2>/dev/null" || echo "") + if [[ -z "$hok" || -z "$wok" ]]; then + log "ERROR: Containers died. Restarting server..." + bash "${SCRIPT_DIR}/start_sglang_multinode.sh" || { log "FATAL: Restart failed"; return 1; } + log "Server restarted." + fi + return 0 +} + +# ================================================================ +# Main +# ================================================================ +main() { + log "=== ${EXPERIMENT} Benchmark ===" + log "Matrix: dsv4_pro6000_vllm_tp_dp_matrix" + log "TTFT SLO: per-ISL (7s-40s), Reps: ${NUM_REPETITIONS}" + log "Server: ${HEAD_IP}:${SGLANG_PORT}, Results: ${RESULT_DIR}" + + if ! curl --fail --silent --max-time 5 "http://${HEAD_IP}:${SGLANG_PORT}/health" >/dev/null 2>&1; then + log "FATAL: Server not healthy"; exit 1 + fi + log "Server health OK." + + local total=0 ok=0 fail=0 oom=0 slo=0 skip=0 + local start_time=$(date +%s) + + for rep in $(seq 1 ${NUM_REPETITIONS}); do + log "========== REPETITION ${rep}/${NUM_REPETITIONS} ==========" + + for isl in "${ISL_LIST[@]}"; do + if ! check_and_recover_server; then exit 1; fi + for osl in "${OSL_LIST[@]}"; do + local tag="${MATRIX[${isl},${osl}]:-N}" + [[ "$tag" == "N" ]] && { ((skip++)); continue; } + log "--- isl=${isl} osl=${osl} [${tag}] ---" + + local conc_list=($(get_conc_for "$isl")) + local combo_oom=0 combo_slo=0 + + for conc in "${conc_list[@]}"; do + [[ $combo_oom -eq 1 ]] && { ((skip++)); continue; } + [[ $combo_slo -eq 1 ]] && { ((slo++)); continue; } + ((total++)) + run_one "$isl" "$osl" "$conc" + case $? in + 0) ((ok++)) ;; + 1) ((fail++)) ;; + 2) ((oom++)); combo_oom=1 ;; + 3) ((slo++)); combo_slo=1 ;; + esac + sleep 2 + done + + local elapsed=$(($(date +%s) - start_time)) + local eta="N/A" + if (( ok > 0 )); then + local rate=$(echo "scale=3; $ok/$elapsed" | bc 2>/dev/null || echo "0") + if [[ "$rate" != "0" ]]; then + local eta_s=$(echo "scale=0; ($total-$ok)/$rate" | bc 2>/dev/null || echo "0") + eta="$(date -u -d @${eta_s} +%Hh%Mm 2>/dev/null || echo '?')" + fi + fi + log "PROGRESS ok=${ok}/${total} fail=${fail} oom=${oom} slo=${slo} elapsed=$(echo "scale=1;$elapsed/3600"|bc)h ETA=${eta}" + done + done + done + + local total_h=$(echo "scale=1; $(($(date +%s)-start_time))/3600" | bc) + log "=== COMPLETE: total=${total} ok=${ok} fail=${fail} oom=${oom} slo=${slo} skip=${skip} time=${total_h}h ===" + log "Results: ${RESULT_DIR}" +} + +main diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_and_bench.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_and_bench.sh new file mode 100755 index 0000000..6acd615 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_and_bench.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } +log "=== Step 1: Starting SGLang multi-node server (EAGLE) ===" +bash "${SCRIPT_DIR}/start_sglang_multinode.sh" +log "=== Step 2: Starting benchmark ===" +bash "${SCRIPT_DIR}/run_batch.sh" +log "=== All done! ===" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_sglang_multinode.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_sglang_multinode.sh new file mode 100755 index 0000000..aa4b1f2 --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_sglang_multinode.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# ============================================================ +# Start SGLang multi-node TP=16 EP=2 + EAGLE speculative decoding +# Usage: ./start_sglang_multinode.sh [--dry-run] +# ============================================================ +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/config.env" + +DRY_RUN="${DRY_RUN:-0}" +HEAD_CONTAINER="${EXPERIMENT}_head" +WORKER_CONTAINER="${EXPERIMENT}_worker" + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } + +build_docker_cmd() { + local node_rank="$1" container_name="$2" + cat </dev/null; +docker run -d --name ${container_name} \\ + --gpus all --network host --ipc=host --ulimit memlock=-1 --shm-size=20g \\ + -v ${MODEL_PATH}:${MODEL_PATH}:ro \\ + -e NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME} \\ + -e NCCL_DEBUG=${NCCL_DEBUG} \\ + -e SGLANG_SHARED_EXPERT_TP1=1 \\ + ${DOCKER_IMAGE} \\ + python3 -m sglang.launch_server \\ + --model-path ${MODEL_PATH} \\ + --tp-size ${TP_SIZE} --ep-size ${EP_SIZE} --nnodes ${NNODES} \\ + --node-rank ${node_rank} --dist-init-addr ${HEAD_IP}:${DIST_INIT_PORT} \\ + --trust-remote-code --host 0.0.0.0 --port ${SGLANG_PORT} \\ + --mem-fraction-static ${MEM_FRACTION_STATIC} \\ + --cuda-graph-max-bs-decode ${CUDA_GRAPH_MAX_BS_DECODE} \\ + --max-running-requests ${MAX_RUNNING_REQUESTS} \\ + --speculative-algorithm ${SPECULATIVE_ALGORITHM} \\ + --speculative-num-steps ${SPECULATIVE_NUM_STEPS} \\ + --speculative-eagle-topk ${SPECULATIVE_EAGLE_TOPK} \\ + --speculative-num-draft-tokens ${SPECULATIVE_NUM_DRAFT_TOKENS} +DOCKER_CMD +} + +main() { + log "=== SGLang TP=${TP_SIZE} EP=${EP_SIZE} + EAGLE ===" + + if [[ "$DRY_RUN" == "0" ]]; then + ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker rm -f ${HEAD_CONTAINER} 2>/dev/null" || true + ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker rm -f ${WORKER_CONTAINER} 2>/dev/null" || true + sleep 2 + fi + + log "Starting WORKER (${WORKER_NODE}, node_rank=1) ..." + local cmd; cmd="$(build_docker_cmd 1 "${WORKER_CONTAINER}")" + if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else + ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "bash -c '${cmd}'" >/dev/null + log "Worker started." + fi + sleep 5 + + log "Starting HEAD (${HEAD_NODE}, node_rank=0) ..." + cmd="$(build_docker_cmd 0 "${HEAD_CONTAINER}")" + if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else + ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "bash -c '${cmd}'" >/dev/null + log "Head started." + fi + + if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] Skip health check"; return 0; fi + log "Waiting for health on ${HEAD_IP}:${SGLANG_PORT} ..." + for (( i=1; i<=${HEALTH_CHECK_RETRIES}; i++ )); do + if curl --fail --silent --max-time 5 "http://${HEAD_IP}:${SGLANG_PORT}/health" >/dev/null 2>&1; then + log "Server healthy! (${i}s)"; return 0 + fi + local hok=$(ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker ps --filter name=${HEAD_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true) + local wok=$(ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker ps --filter name=${WORKER_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true) + if [[ -z "$hok" ]]; then log "ERROR: Head died"; ssh "${HEAD_NODE}" "docker logs --tail 30 ${HEAD_CONTAINER}" || true; return 1; fi + if [[ -z "$wok" ]]; then log "ERROR: Worker died"; ssh "${WORKER_NODE}" "docker logs --tail 30 ${WORKER_CONTAINER}" || true; return 1; fi + sleep "${HEALTH_CHECK_INTERVAL_S}" + done + log "ERROR: Timeout"; return 1 +} + +main "$@" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/stop_sglang_multinode.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/stop_sglang_multinode.sh new file mode 100755 index 0000000..f9bf14b --- /dev/null +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/stop_sglang_multinode.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/config.env" +log() { echo "[$(date '+%H:%M:%S')] $*"; } +log "Stopping ${EXPERIMENT} ..." +ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker rm -f ${EXPERIMENT}_head 2>/dev/null && echo 'Head stopped' || echo 'Head already stopped'" +ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker rm -f ${EXPERIMENT}_worker 2>/dev/null && echo 'Worker stopped' || echo 'Worker already stopped'" +log "Done."