tune for 1 orchestrator + up to 3 subagents: gpu-mem 0.88 (~15GB reserve), seqs=4, ctx 262144; document use case + tradeoffs

2026-06-24 13:31:36 +10:00
parent 6797a9f6dd
commit dabc866507
4 changed files with 30 additions and 17 deletions
@@ -8,7 +8,7 @@ NSPEC="${1:-2}"
 BACKEND="${2:-flash_attn}"
 MODEL="${MODEL:-Intel/Qwen3.5-122B-A10B-int4-AutoRound}"
 MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
-GPU_MEM="${GPU_MEM:-0.89}"
+GPU_MEM="${GPU_MEM:-0.88}"
 MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}"
 MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"
 PORT="${PORT:-8000}"