VALIDATED defaults: gpu-mem 0.82 + seqs 3 (0.88+ over-subscribes/swaps); README corrected with measured KV pool (457k tok, 1.74x) + concurrency curve

2026-06-24 14:30:56 +10:00
parent dabc866507
commit df8981d54c
4 changed files with 35 additions and 33 deletions
@@ -8,8 +8,8 @@ NSPEC="${1:-2}"
 BACKEND="${2:-flash_attn}"
 MODEL="${MODEL:-Intel/Qwen3.5-122B-A10B-int4-AutoRound}"
 MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
-GPU_MEM="${GPU_MEM:-0.88}"
-MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}"
+GPU_MEM="${GPU_MEM:-0.82}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-3}"
 MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"
 PORT="${PORT:-8000}"
 echo "[mtp] qwen3_5_mtp — backend=$BACKEND, num_speculative_tokens=$NSPEC, model=$MODEL"