diff --git a/README.md b/README.md index c192142..06e3a5e 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,18 @@ streams queue rather than fail) or a smaller `--ctx`. > [`scripts/monitor.sh`](scripts/monitor.sh) (OOM auto-kill guard) the first time > at any new `gpu-mem` or `ctx`. +### Startup time + +Time to READY is **~3 min** (vs ~12 min on the default loader). `--load-format +fastsafetensors` (default) reads the 62–71 GiB **straight to the device** — +**~32 s** vs ~8 min for the default mmap-backed per-tensor read + redundant +CPU→GPU copy, which is pathologically slow on GB10's current kernel (the copy is +also physically pointless on unified memory). It falls back to `nogds` +automatically — no GPUDirect Storage hardware required; override with +`LOAD_FORMAT=auto`. The remaining ~2.5 min is `torch.compile` (~42 s) + CUDA-graph +capture (~2 min); the compile cache is persisted at `$HF_HOME/.vllm_cache`, so +boots after the first skip the compile. + ## Profiles Selected with `--profile`: diff --git a/install.sh b/install.sh index 8e70aef..7d24cbe 100755 --- a/install.sh +++ b/install.sh @@ -288,7 +288,7 @@ start_server() { docker rm -f "$NAME" >/dev/null 2>&1 || true # shellcheck disable=SC2086 docker run -d --name "$NAME" --gpus all --net=host --ipc=host --ulimit memlock=-1:-1 \ - -e HF_HOME=/hf -e MAX_MODEL_LEN="$CTX" -e GPU_MEM="$GPU_MEM" \ + -e HF_HOME=/hf -e VLLM_CACHE_ROOT=/hf/.vllm_cache -e MAX_MODEL_LEN="$CTX" -e GPU_MEM="$GPU_MEM" \ -e MAX_NUM_SEQS="$MAX_NUM_SEQS" -e MAX_BATCHED_TOKENS="$MAX_BATCHED_TOKENS" ${HF_TOKEN:+-e HF_TOKEN="$HF_TOKEN"} \ "${model_env[@]}" \ -v "$HF_HOME:/hf" -v "$REPO_DIR/runtime:/host:ro" "${mounts[@]}" \ diff --git a/runtime/mtp_serve.sh b/runtime/mtp_serve.sh index 6f76985..4c01db6 100755 --- a/runtime/mtp_serve.sh +++ b/runtime/mtp_serve.sh @@ -11,6 +11,7 @@ MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}" GPU_MEM="${GPU_MEM:-0.82}" MAX_NUM_SEQS="${MAX_NUM_SEQS:-3}" MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}" +LOAD_FORMAT="${LOAD_FORMAT:-fastsafetensors}" PORT="${PORT:-8000}" echo "[mtp] qwen3_5_mtp — backend=$BACKEND, num_speculative_tokens=$NSPEC, model=$MODEL" exec vllm serve "$MODEL" \ @@ -23,5 +24,6 @@ exec vllm serve "$MODEL" \ --no-enable-prefix-caching \ --enable-chunked-prefill \ --trust-remote-code \ + --load-format "$LOAD_FORMAT" \ --attention-backend "$BACKEND" \ --speculative-config "{\"method\":\"qwen3_5_mtp\",\"num_speculative_tokens\":$NSPEC,\"model\":\"$MODEL\"}"