ent
39 lines
1.5 KiB
Bash
Executable File
39 lines
1.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Monitor SGLang container startup with an OOM auto-kill guard.
|
|
# Breaks on: READY, ERROR (traceback), OOM-GUARD (avail mem too low), or timeout.
|
|
set -u
|
|
NAME="${1:-sglang-dflash}"
|
|
MAX_ITERS="${2:-60}" # 60 * 20s = 20 min
|
|
FLOOR_MB="${3:-4096}" # kill if available memory drops below this
|
|
|
|
for i in $(seq 1 "$MAX_ITERS"); do
|
|
if ! docker ps --filter "name=$NAME" --format '{{.Names}}' | grep -q "$NAME"; then
|
|
echo "STATE=EXITED iter=$i"
|
|
echo "--- last log ---"; docker logs "$NAME" 2>&1 | tail -25
|
|
exit 0
|
|
fi
|
|
AVAIL=$(free -m | awk '/^Mem:/{print $7}')
|
|
if [ "$AVAIL" -lt "$FLOOR_MB" ]; then
|
|
echo "STATE=OOM-GUARD iter=$i avail_mb=$AVAIL -> killing $NAME"
|
|
docker kill "$NAME" >/dev/null 2>&1
|
|
echo "--- last log ---"; docker logs "$NAME" 2>&1 | tail -25
|
|
exit 0
|
|
fi
|
|
LOG=$(docker logs "$NAME" 2>&1)
|
|
if echo "$LOG" | grep -qiE "server is fired up|Application startup complete|The server is ready"; then
|
|
echo "STATE=READY iter=$i avail_mb=$AVAIL"
|
|
echo "--- tail ---"; echo "$LOG" | tail -20
|
|
exit 0
|
|
fi
|
|
if echo "$LOG" | grep -qE "Traceback \(most recent call last\)|CUDA out of memory|RuntimeError|AssertionError|ValueError|raise NotImplementedError"; then
|
|
echo "STATE=ERROR iter=$i avail_mb=$AVAIL"
|
|
echo "--- tail ---"; echo "$LOG" | tail -40
|
|
exit 0
|
|
fi
|
|
LAST=$(echo "$LOG" | tail -1 | cut -c1-110)
|
|
echo "iter=$i avail_mb=$AVAIL :: $LAST"
|
|
sleep 20
|
|
done
|
|
echo "STATE=TIMEOUT after $MAX_ITERS iters"
|
|
docker logs "$NAME" 2>&1 | tail -20
|