TBNilles/qwen3.5-122B-A10B-on-spark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

load speed: persist compile cache at $HF_HOME/.vllm_cache (was wiped each container boot, -42s/boot); propagate --load-format to mtp_serve; document startup time

Browse Source
This commit is contained in:
ent
2026-06-24 16:32:02 +10:00
parent bdb57d776d
commit d3688cc833
3 changed files with 15 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+12
View File
@@ -129,6 +129,18 @@ streams queue rather than fail) or a smaller `--ctx`.
> [`scripts/monitor.sh`](scripts/monitor.sh) (OOM auto-kill guard) the first time > [`scripts/monitor.sh`](scripts/monitor.sh) (OOM auto-kill guard) the first time
> at any new `gpu-mem` or `ctx`. > at any new `gpu-mem` or `ctx`.
### Startup time
Time to READY is **~3 min** (vs ~12 min on the default loader). `--load-format
fastsafetensors` (default) reads the 62–71 GiB **straight to the device** —
**~32 s** vs ~8 min for the default mmap-backed per-tensor read + redundant
CPU→GPU copy, which is pathologically slow on GB10's current kernel (the copy is
also physically pointless on unified memory). It falls back to `nogds`
automatically — no GPUDirect Storage hardware required; override with
`LOAD_FORMAT=auto`. The remaining ~2.5 min is `torch.compile` (~42 s) + CUDA-graph
capture (~2 min); the compile cache is persisted at `$HF_HOME/.vllm_cache`, so
boots after the first skip the compile.
## Profiles ## Profiles
Selected with `--profile`: Selected with `--profile`:
install.sh
+1 -1
View File
@@ -288,7 +288,7 @@ start_server() {
docker rm -f "$NAME" >/dev/null 2>&1 || true docker rm -f "$NAME" >/dev/null 2>&1 || true
# shellcheck disable=SC2086 # shellcheck disable=SC2086
docker run -d --name "$NAME" --gpus all --net=host --ipc=host --ulimit memlock=-1:-1 \ docker run -d --name "$NAME" --gpus all --net=host --ipc=host --ulimit memlock=-1:-1 \
-e HF_HOME=/hf -e MAX_MODEL_LEN="$CTX" -e GPU_MEM="$GPU_MEM" \ -e HF_HOME=/hf -e VLLM_CACHE_ROOT=/hf/.vllm_cache -e MAX_MODEL_LEN="$CTX" -e GPU_MEM="$GPU_MEM" \
-e MAX_NUM_SEQS="$MAX_NUM_SEQS" -e MAX_BATCHED_TOKENS="$MAX_BATCHED_TOKENS" ${HF_TOKEN:+-e HF_TOKEN="$HF_TOKEN"} \ -e MAX_NUM_SEQS="$MAX_NUM_SEQS" -e MAX_BATCHED_TOKENS="$MAX_BATCHED_TOKENS" ${HF_TOKEN:+-e HF_TOKEN="$HF_TOKEN"} \
"${model_env[@]}" \ "${model_env[@]}" \
-v "$HF_HOME:/hf" -v "$REPO_DIR/runtime:/host:ro" "${mounts[@]}" \ -v "$HF_HOME:/hf" -v "$REPO_DIR/runtime:/host:ro" "${mounts[@]}" \
runtime/mtp_serve.sh
+2
View File
@@ -11,6 +11,7 @@ MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
GPU_MEM="${GPU_MEM:-0.82}" GPU_MEM="${GPU_MEM:-0.82}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-3}" MAX_NUM_SEQS="${MAX_NUM_SEQS:-3}"
MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}" MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"
LOAD_FORMAT="${LOAD_FORMAT:-fastsafetensors}"
PORT="${PORT:-8000}" PORT="${PORT:-8000}"
echo "[mtp] qwen3_5_mtp — backend=$BACKEND, num_speculative_tokens=$NSPEC, model=$MODEL" echo "[mtp] qwen3_5_mtp — backend=$BACKEND, num_speculative_tokens=$NSPEC, model=$MODEL"
exec vllm serve "$MODEL" \ exec vllm serve "$MODEL" \
@@ -23,5 +24,6 @@ exec vllm serve "$MODEL" \
--no-enable-prefix-caching \ --no-enable-prefix-caching \
--enable-chunked-prefill \ --enable-chunked-prefill \
--trust-remote-code \ --trust-remote-code \
--load-format "$LOAD_FORMAT" \
--attention-backend "$BACKEND" \ --attention-backend "$BACKEND" \
--speculative-config "{\"method\":\"qwen3_5_mtp\",\"num_speculative_tokens\":$NSPEC,\"model\":\"$MODEL\"}" --speculative-config "{\"method\":\"qwen3_5_mtp\",\"num_speculative_tokens\":$NSPEC,\"model\":\"$MODEL\"}"