From 6797a9f6dda2e9a9535471dad39682a3674c4bd7 Mon Sep 17 00:00:00 2001
From: ent <bleysg@gmail.com>
Date: Wed, 24 Jun 2026 13:21:52 +1000
Subject: [PATCH] default max-num-seqs=4 (>=2 with headroom; ~5 fit full native
 ctx each)

---
 README.md            | 2 +-
 install.sh           | 4 ++--
 runtime/mtp_serve.sh | 2 +-
 runtime/serve.sh     | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 0c5d8ed..9f69ee9 100644
--- a/README.md
+++ b/README.md
@@ -90,7 +90,7 @@ capped by the *model*, not memory. Defaults (override via flags/env):
 |---|---|---|
 | `--gpu-mem` / `GPU_MEM` | **0.89** | reserves ~14 GB; drop to 0.87 if the OOM-guard fires on first load |
 | `--ctx` / `CTX` (`MAX_MODEL_LEN`) | **262144** | model native max |
-| `--max-num-seqs` / `MAX_NUM_SEQS` | **1** | single-stream; raising it is nearly free (pool ≫ one context) |
+| `--max-num-seqs` / `MAX_NUM_SEQS` | **4** | ~5 sequences fit the full 262 144 context each (pool ÷ ctx); set higher for short-context concurrency (preempts if many grow long) |
 | `--max-batched-tokens` / `MAX_BATCHED_TOKENS` | **8192** | chunked-prefill chunk — kept **below** ctx so a long prefill doesn't batch all at once |
 
 > Unified-memory OOM **hard-freezes** the box, and vLLM's profiler can undershoot
diff --git a/install.sh b/install.sh
index 959f494..de5fdc2 100755
--- a/install.sh
+++ b/install.sh
@@ -57,7 +57,7 @@ NSPEC=""                           # override num_speculative_tokens (default pe
 PORT="${PORT:-8000}"
 CTX="${CTX:-262144}"               # max-model-len: model native max (KV is ~24 KiB/token)
 GPU_MEM="${GPU_MEM:-0.89}"         # reserves ~14 GB on a 128 GB (119 GiB) GB10
-MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"          # single-stream; raise for concurrency (cheap here)
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}"          # ~5 fit full native ctx each; raise for short-ctx concurrency
 MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"  # chunked-prefill chunk (decoupled from ctx)
 BACKEND="${BACKEND:-flash_attn}"
 
@@ -95,7 +95,7 @@ Flags:
   --port N                Server port (default: $PORT).
   --ctx N                 max-model-len (default: $CTX = model native max).
   --gpu-mem F             gpu-memory-utilization (default: $GPU_MEM reserves ~14 GB on 119 GiB).
-  --max-num-seqs N        concurrent sequences (default: $MAX_NUM_SEQS, single-stream).
+  --max-num-seqs N        concurrent sequences (default: $MAX_NUM_SEQS; ~5 fit full ctx each).
   --max-batched-tokens N  chunked-prefill chunk (default: $MAX_BATCHED_TOKENS; keep < ctx).
   --force                 Skip the GB10/SM121 host check.
   --no-smoke              Start the server but skip the Paris smoke test.
diff --git a/runtime/mtp_serve.sh b/runtime/mtp_serve.sh
index 140e6aa..284e6ba 100755
--- a/runtime/mtp_serve.sh
+++ b/runtime/mtp_serve.sh
@@ -9,7 +9,7 @@ BACKEND="${2:-flash_attn}"
 MODEL="${MODEL:-Intel/Qwen3.5-122B-A10B-int4-AutoRound}"
 MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
 GPU_MEM="${GPU_MEM:-0.89}"
-MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}"
 MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"
 PORT="${PORT:-8000}"
 echo "[mtp] qwen3_5_mtp — backend=$BACKEND, num_speculative_tokens=$NSPEC, model=$MODEL"
diff --git a/runtime/serve.sh b/runtime/serve.sh
index 89ef9f7..dfea383 100755
--- a/runtime/serve.sh
+++ b/runtime/serve.sh
@@ -21,7 +21,7 @@ MODEL="${MODEL:-Intel/Qwen3.5-122B-A10B-int4-AutoRound}"
 DRAFT="${DRAFT:-z-lab/Qwen3.5-122B-A10B-DFlash}"
 MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"      # model native max; KV is ~24 KiB/token so it fits
 GPU_MEM="${GPU_MEM:-0.89}"                     # ~14 GB reserved on a 128 GB (119 GiB) GB10
-MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"             # single-stream; each seq also reserves GDN state
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}"             # ~5 seqs fit the full native ctx each (KV pool ~1.38M / 262144)
 MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"  # chunked-prefill chunk (NOT = max-model-len)
 PORT="${PORT:-8000}"