TBNilles/qwen3.5-122B-A10B-on-spark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tune for 1 orchestrator + up to 3 subagents: gpu-mem 0.88 (~15GB reserve), seqs=4, ctx 262144; document use case + tradeoffs

Browse Source
This commit is contained in:
ent
2026-06-24 13:31:36 +10:00
parent 6797a9f6dd
commit dabc866507
4 changed files with 30 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/mtp_serve.sh
+1 -1
View File
@@ -8,7 +8,7 @@ NSPEC="${1:-2}"
BACKEND="${2:-flash_attn}"
MODEL="${MODEL:-Intel/Qwen3.5-122B-A10B-int4-AutoRound}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
GPU_MEM="${GPU_MEM:-0.89}"
GPU_MEM="${GPU_MEM:-0.88}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}"
MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"
PORT="${PORT:-8000}"
runtime/serve.sh
+1 -1
View File
@@ -20,7 +20,7 @@ BACKEND="${2:-flash_attn}"
MODEL="${MODEL:-Intel/Qwen3.5-122B-A10B-int4-AutoRound}"
DRAFT="${DRAFT:-z-lab/Qwen3.5-122B-A10B-DFlash}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}" # model native max; KV is ~24 KiB/token so it fits
GPU_MEM="${GPU_MEM:-0.89}" # ~14 GB reserved on a 128 GB (119 GiB) GB10
GPU_MEM="${GPU_MEM:-0.88}" # ~15 GB reserved on a 128 GB (119 GiB) GB10
MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}" # ~5 seqs fit the full native ctx each (KV pool ~1.38M / 262144)
MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}" # chunked-prefill chunk (NOT = max-model-len)
PORT="${PORT:-8000}"