diff --git a/README.md b/README.md index 8318527..462fd6c 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,17 @@ Selected with `--profile`: The server is OpenAI-compatible (`/v1/chat/completions` with tool calls + SSE, `/v1/completions`, `/v1/models`) and serves under the model name `qwen`. +**Tool calling is enabled by default** (`--enable-auto-tool-choice +--tool-call-parser qwen3_xml --reasoning-parser qwen3`), so clients can send `tools` +with `tool_choice="auto"`. Qwen3.5 emits the *XML* tool format, so `qwen3_xml` is the +correct parser — `hermes` returns 200 but with empty `tool_calls` (the call lands in +`content` instead). `` reasoning is split into `reasoning_content`. Override +with `TOOL_PARSER=` / `REASONING_PARSER=` (empty disables; `qwen3_coder` also works). + +**Network:** the server binds `0.0.0.0` in `--net=host`, so it is reachable from the +LAN at `http://:8000` — the `127.0.0.1` in the examples is just the local +default. There is no auth; put it behind a reverse proxy / firewall for shared use. + ## Benchmarks All single-stream (c=1), temperature 0, GB10. "Hermes" regenerates the next diff --git a/runtime/mtp_serve.sh b/runtime/mtp_serve.sh index 5223378..3112c3b 100755 --- a/runtime/mtp_serve.sh +++ b/runtime/mtp_serve.sh @@ -15,6 +15,12 @@ LOAD_FORMAT="${LOAD_FORMAT:-fastsafetensors}" PORT="${PORT:-8000}" # Reclaim the CUDA-graph memory over-estimate to KV (see serve.sh). Set =1 to restore. export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS="${VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS:-0}" +# Auto tool-calling + reasoning split (see serve.sh). Off via TOOL_PARSER="" etc. +TOOL_PARSER="${TOOL_PARSER:-qwen3_xml}" +REASONING_PARSER="${REASONING_PARSER:-qwen3}" +TOOL_ARG=() +[ -n "$TOOL_PARSER" ] && TOOL_ARG+=(--enable-auto-tool-choice --tool-call-parser "$TOOL_PARSER") +[ -n "$REASONING_PARSER" ] && TOOL_ARG+=(--reasoning-parser "$REASONING_PARSER") echo "[mtp] qwen3_5_mtp — backend=$BACKEND, num_speculative_tokens=$NSPEC, model=$MODEL" exec vllm serve "$MODEL" \ --served-model-name qwen \ @@ -28,4 +34,5 @@ exec vllm serve "$MODEL" \ --trust-remote-code \ --load-format "$LOAD_FORMAT" \ --attention-backend "$BACKEND" \ + "${TOOL_ARG[@]}" \ --speculative-config "{\"method\":\"qwen3_5_mtp\",\"num_speculative_tokens\":$NSPEC,\"model\":\"$MODEL\"}" diff --git a/runtime/serve.sh b/runtime/serve.sh index 05d6c57..63333ee 100755 --- a/runtime/serve.sh +++ b/runtime/serve.sh @@ -60,6 +60,22 @@ else fi python3 /host/patch_unify2.py || { [ "$NSPEC" = "0" ] && true; } +# OpenAI automatic tool-calling + reasoning split. Without --enable-auto-tool-choice +# + --tool-call-parser, any client that sends `tools` with tool_choice="auto" gets +# HTTP 400 — so agent/tool workloads (the whole point here) need these on. The tool +# parser only activates when a request carries `tools`, so leaving it on is free for +# plain chat. Qwen3.5 emits the XML tool format +# (v) -> the +# qwen3_xml parser (NOT hermes, which only reads the JSON format -> empty tool_calls), +# and reasoning the qwen3 parser splits into `reasoning_content`. Verified: +# tool_choice="auto" -> tool_calls=[get_weather {"city":"Paris"}]. Disable either with +# TOOL_PARSER="" / REASONING_PARSER="", or override the name (e.g. qwen3_coder). +TOOL_PARSER="${TOOL_PARSER:-qwen3_xml}" +REASONING_PARSER="${REASONING_PARSER:-qwen3}" +TOOL_ARG=() +[ -n "$TOOL_PARSER" ] && TOOL_ARG+=(--enable-auto-tool-choice --tool-call-parser "$TOOL_PARSER") +[ -n "$REASONING_PARSER" ] && TOOL_ARG+=(--reasoning-parser "$REASONING_PARSER") + exec vllm serve "$MODEL" \ --served-model-name qwen \ --host 0.0.0.0 --port "$PORT" \ @@ -72,4 +88,5 @@ exec vllm serve "$MODEL" \ --trust-remote-code \ --load-format "$LOAD_FORMAT" \ --attention-backend "$BACKEND" \ + "${TOOL_ARG[@]}" \ "${SPEC_ARG[@]}"