From bdb57d776d0836e639ed7ed824cc394fad5ff501 Mon Sep 17 00:00:00 2001
From: ent <bleysg@gmail.com>
Date: Wed, 24 Jun 2026 16:30:33 +1000
Subject: [PATCH] =?UTF-8?q?load=20speed:=20--load-format=20fastsafetensors?=
 =?UTF-8?q?=20(read=20straight=20to=20device,=20no=20mmap)=20=E2=80=94=20w?=
 =?UTF-8?q?eight=20load=20463s=20->=2032s=20(14.5x),=20time-to-READY=20~12?=
 =?UTF-8?q?min=20->=20~3min=20on=20GB10.=20Falls=20back=20to=20nogds=20aut?=
 =?UTF-8?q?omatically.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 runtime/serve.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/runtime/serve.sh b/runtime/serve.sh
index 1305c60..5451514 100755
--- a/runtime/serve.sh
+++ b/runtime/serve.sh
@@ -24,6 +24,11 @@ GPU_MEM="${GPU_MEM:-0.82}"                     # VALIDATED: ~14 GiB free on 128
 MAX_NUM_SEQS="${MAX_NUM_SEQS:-3}"             # 3 concurrent streams; KV pool ~457k tokens, 1.74x at full 262144
 MAX_BATCHED_TOKENS="${MAX_BATCHED_TOKENS:-8192}"  # chunked-prefill chunk (NOT = max-model-len)
 PORT="${PORT:-8000}"
+# Read straight to the device (no mmap, no host staging) — the slow default safetensors
+# read+copy is ~8 min on Spark; fastsafetensors cuts it to ~1 min. Falls back to nogds
+# automatically. Override LOAD_FORMAT=auto|safetensors if the pkg is absent (or set
+# --safetensors-load-strategy eager via SAFETENSORS_STRATEGY).
+LOAD_FORMAT="${LOAD_FORMAT:-fastsafetensors}"
 
 # FLA sm121 big-tile shmem fix (prefill/TTFT only on sm121; harmless, free).
 echo "[serve] FLA sm121 big-tile shmem patch"
@@ -57,5 +62,6 @@ exec vllm serve "$MODEL" \
   --no-enable-prefix-caching \
   --enable-chunked-prefill \
   --trust-remote-code \
+  --load-format "$LOAD_FORMAT" \
   --attention-backend "$BACKEND" \
   "${SPEC_ARG[@]}"