TBNilles/qwen3.5-122B-A10B-on-spark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tune defaults for max context + KV depth: gpu-mem 0.89 (reserve ~14GB), ctx 262144, max-num-seqs 1, decouple max-batched-tokens (8192)

Browse Source
This commit is contained in:
ent
2026-06-24 13:18:32 +10:00
parent 70c91f94a7
commit 7dd6a74d38
4 changed files with 55 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/mtp_serve.sh
+6 -4
View File
@@ -7,16 +7,18 @@ set -euo pipefail
NSPEC="${1:-2}"
BACKEND="${2:-flash_attn}"
MODEL="${MODEL:-Intel/Qwen3.5-122B-A10B-int4-AutoRound}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}"
GPU_MEM="${GPU_MEM:-0.8}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
GPU_MEM="${GPU_MEM:-0.89}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"
MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"
PORT="${PORT:-8000}"
echo "[mtp] qwen3_5_mtp — backend=$BACKEND, num_speculative_tokens=$NSPEC, model=$MODEL"
exec vllm serve "$MODEL" \
--served-model-name qwen \
--host 0.0.0.0 --port "$PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--max-num-seqs 16 \
--max-num-batched-tokens "$MAX_MODEL_LEN" \
--max-num-seqs "$MAX_NUM_SEQS" \
--max-num-batched-tokens "$MAX_BATCHED_TOKENS" \
--gpu-memory-utilization "$GPU_MEM" \
--no-enable-prefix-caching \
--enable-chunked-prefill \
runtime/serve.sh
+6 -4
View File
@@ -19,8 +19,10 @@ NSPEC="${1:-12}"
BACKEND="${2:-flash_attn}"
MODEL="${MODEL:-Intel/Qwen3.5-122B-A10B-int4-AutoRound}"
DRAFT="${DRAFT:-z-lab/Qwen3.5-122B-A10B-DFlash}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}"
GPU_MEM="${GPU_MEM:-0.8}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}" # model native max; KV is ~24 KiB/token so it fits
GPU_MEM="${GPU_MEM:-0.89}" # ~14 GB reserved on a 128 GB (119 GiB) GB10
MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}" # single-stream; each seq also reserves GDN state
MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}" # chunked-prefill chunk (NOT = max-model-len)
PORT="${PORT:-8000}"
# FLA sm121 big-tile shmem fix (prefill/TTFT only on sm121; harmless, free).
@@ -49,8 +51,8 @@ exec vllm serve "$MODEL" \
--served-model-name qwen \
--host 0.0.0.0 --port "$PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--max-num-seqs 16 \
--max-num-batched-tokens "$MAX_MODEL_LEN" \
--max-num-seqs "$MAX_NUM_SEQS" \
--max-num-batched-tokens "$MAX_BATCHED_TOKENS" \
--gpu-memory-utilization "$GPU_MEM" \
--no-enable-prefix-caching \
--enable-chunked-prefill \