load speed: persist compile cache at $HF_HOME/.vllm_cache (was wiped each container boot, -42s/boot); propagate --load-format to mtp_serve; document startup time

2026-06-24 16:32:02 +10:00
parent bdb57d776d
commit d3688cc833
3 changed files with 15 additions and 1 deletions
@@ -288,7 +288,7 @@ start_server() {
    docker rm -f "$NAME" >/dev/null 2>&1 || true
    # shellcheck disable=SC2086
    docker run -d --name "$NAME" --gpus all --net=host --ipc=host --ulimit memlock=-1:-1 \
-        -e HF_HOME=/hf -e MAX_MODEL_LEN="$CTX" -e GPU_MEM="$GPU_MEM" \
+        -e HF_HOME=/hf -e VLLM_CACHE_ROOT=/hf/.vllm_cache -e MAX_MODEL_LEN="$CTX" -e GPU_MEM="$GPU_MEM" \
        -e MAX_NUM_SEQS="$MAX_NUM_SEQS" -e MAX_BATCHED_TOKENS="$MAX_BATCHED_TOKENS" ${HF_TOKEN:+-e HF_TOKEN="$HF_TOKEN"} \
        "${model_env[@]}" \
        -v "$HF_HOME:/hf" -v "$REPO_DIR/runtime:/host:ro" "${mounts[@]}" \