default max-num-seqs=4 (>=2 with headroom; ~5 fit full native ctx each)

2026-06-24 13:21:52 +10:00
parent 7dd6a74d38
commit 6797a9f6dd
4 changed files with 5 additions and 5 deletions
@@ -57,7 +57,7 @@ NSPEC=""                           # override num_speculative_tokens (default pe
 PORT="${PORT:-8000}"
 CTX="${CTX:-262144}"               # max-model-len: model native max (KV is ~24 KiB/token)
 GPU_MEM="${GPU_MEM:-0.89}"         # reserves ~14 GB on a 128 GB (119 GiB) GB10
-MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"          # single-stream; raise for concurrency (cheap here)
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}"          # ~5 fit full native ctx each; raise for short-ctx concurrency
 MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"  # chunked-prefill chunk (decoupled from ctx)
 BACKEND="${BACKEND:-flash_attn}"

@@ -95,7 +95,7 @@ Flags:
  --port N                Server port (default: $PORT).
  --ctx N                 max-model-len (default: $CTX = model native max).
  --gpu-mem F             gpu-memory-utilization (default: $GPU_MEM reserves ~14 GB on 119 GiB).
-  --max-num-seqs N        concurrent sequences (default: $MAX_NUM_SEQS, single-stream).
+  --max-num-seqs N        concurrent sequences (default: $MAX_NUM_SEQS; ~5 fit full ctx each).
  --max-batched-tokens N  chunked-prefill chunk (default: $MAX_BATCHED_TOKENS; keep < ctx).
  --force                 Skip the GB10/SM121 host check.
  --no-smoke              Start the server but skip the Paris smoke test.