qwen3.5-122B-A10B on DGX Spark: vLLM + DFlash + dense-bandwidth stack, one-shot installer
This commit is contained in:
ent
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Faithful reproduction of albond's bench_qwen35.sh methodology so our DFlash/MTP
|
||||
numbers are directly comparable to his reported 51.58 tok/s.
|
||||
|
||||
His method (verbatim): non-streaming /v1/chat/completions, time the WHOLE request,
|
||||
tok/s = completion_tokens / wall_time (INCLUDES prefill + overhead = END-TO-END).
|
||||
5 prompts (Q&A 256 / Code 512 / JSON 1024 / Math 64 / LongCode 2048), temp 0, run 1
|
||||
discarded as JIT warmup. We add: decode-only tok/s isn't measured here on purpose
|
||||
(his isn't either) + an aggregate spec-accept-len from /metrics deltas for context.
|
||||
"""
|
||||
import json, sys, time, urllib.request, statistics
|
||||
|
||||
BASE = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8000"
|
||||
MODEL = "qwen"
|
||||
PROMPTS = [
|
||||
("Q&A", "What are the main differences between TCP and UDP? Be concise.", 256),
|
||||
("Code", "Write a Python function that implements binary search on a sorted list. Include type hints and docstring.", 512),
|
||||
("JSON", "Generate a JSON array of 10 fictional employees with fields: name, age, department, salary, email, skills (array of 3). Output ONLY valid JSON, no explanation.", 1024),
|
||||
("Math", "What is 7823 * 4519? Show only the answer.", 64),
|
||||
("LongCode", "Write a complete Python implementation of a red-black tree with insert, delete, search, and in-order traversal. Include all rotation methods.", 2048),
|
||||
]
|
||||
|
||||
|
||||
def scrape():
|
||||
acc = dr = 0.0
|
||||
try:
|
||||
txt = urllib.request.urlopen(BASE + "/metrics", timeout=10).read().decode()
|
||||
for ln in txt.splitlines():
|
||||
if ln.startswith("#") or not ln.split():
|
||||
continue
|
||||
v = float(ln.split()[-1])
|
||||
if "spec_decode_num_accepted_tokens_total" in ln:
|
||||
acc += v
|
||||
elif "spec_decode_num_drafts_total" in ln:
|
||||
dr += v
|
||||
except Exception:
|
||||
pass
|
||||
return acc, dr
|
||||
|
||||
|
||||
def chat_e2e(prompt, max_tokens):
|
||||
body = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens, "temperature": 0.0}).encode()
|
||||
req = urllib.request.Request(BASE + "/v1/chat/completions", data=body,
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer x"})
|
||||
t0 = time.perf_counter()
|
||||
r = json.loads(urllib.request.urlopen(req, timeout=600).read())
|
||||
elapsed = time.perf_counter() - t0
|
||||
ct = r["usage"]["completion_tokens"]
|
||||
return ct, elapsed, ct / elapsed if elapsed > 0 else 0.0
|
||||
|
||||
|
||||
def main():
|
||||
label = sys.argv[2] if len(sys.argv) > 2 else "server"
|
||||
print(f"=== albond-method e2e bench :: {label} ===")
|
||||
a0, d0 = scrape()
|
||||
run2 = {}
|
||||
for run in (1, 2):
|
||||
tag = "WARMUP(discard)" if run == 1 else "RUN2"
|
||||
for name, prompt, mt in PROMPTS:
|
||||
try:
|
||||
ct, el, tps = chat_e2e(prompt, mt)
|
||||
except Exception as e:
|
||||
print(f" [{name}] FAILED: {type(e).__name__}: {e}"); continue
|
||||
if run == 2:
|
||||
run2[name] = tps
|
||||
print(f" {tag:16s} [{name:8s}] {ct:4d} tok in {el:6.2f}s = {tps:6.1f} tok/s (e2e)")
|
||||
a1, d1 = scrape()
|
||||
print(f"\n{label} RUN2 e2e tok/s: " + " ".join(f"{k}={v:.1f}" for k, v in run2.items()))
|
||||
if run2:
|
||||
print(f" cross-prompt mean (RUN2) = {statistics.mean(run2.values()):.1f} tok/s (albond reports 51.58)")
|
||||
if d1 - d0 > 0:
|
||||
print(f" aggregate spec accept length over bench = {1 + (a1-a0)/(d1-d0):.2f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Single-stream (c=1) decode-rate benchmark for an OpenAI-compatible vLLM server.
|
||||
|
||||
Measures *decode* tok/s (excludes TTFT/prefill) over N sequential requests, and
|
||||
optionally scrapes vLLM /metrics for speculative-decode acceptance length.
|
||||
|
||||
stdlib only — runs anywhere with python3, no pip installs.
|
||||
|
||||
python3 bench_decode.py --base-url http://127.0.0.1:8000 \
|
||||
--model Intel/Qwen3.5-122B-A10B-int4-AutoRound \
|
||||
--max-tokens 256 --runs 5 --label "int4 baseline"
|
||||
"""
|
||||
import argparse, json, statistics, sys, time, urllib.request, urllib.error
|
||||
|
||||
PROMPT = ("You are a careful writer. Write a detailed, continuous explanation of how "
|
||||
"a modern mixture-of-experts transformer performs autoregressive decoding, "
|
||||
"covering routing, KV cache, and memory bandwidth. Begin now:\n\n")
|
||||
|
||||
|
||||
def post_stream(base_url, model, prompt, max_tokens, timeout):
|
||||
"""Stream /v1/completions; return (completion_tokens, t_first, t_last)."""
|
||||
body = json.dumps({
|
||||
"model": model, "prompt": prompt, "max_tokens": max_tokens,
|
||||
"temperature": 0.0, "stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
# force the full token budget so we measure steady-state decode
|
||||
"ignore_eos": True, "min_tokens": max_tokens,
|
||||
}).encode()
|
||||
req = urllib.request.Request(base_url.rstrip("/") + "/v1/completions",
|
||||
data=body, headers={"Content-Type": "application/json",
|
||||
"Authorization": "Bearer x"})
|
||||
t_first = t_last = None
|
||||
completion_tokens = 0
|
||||
chunks = 0
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
for raw in r:
|
||||
line = raw.decode("utf-8", "replace").strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data = line[5:].strip()
|
||||
if data == "[DONE]":
|
||||
break
|
||||
try:
|
||||
obj = json.loads(data)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
usage = obj.get("usage")
|
||||
if usage and usage.get("completion_tokens"):
|
||||
completion_tokens = usage["completion_tokens"]
|
||||
choices = obj.get("choices") or []
|
||||
if choices and choices[0].get("text"):
|
||||
now = time.perf_counter()
|
||||
if t_first is None:
|
||||
t_first = now
|
||||
t_last = now
|
||||
chunks += 1
|
||||
if completion_tokens == 0:
|
||||
completion_tokens = chunks # fallback: 1 chunk ~= 1 token
|
||||
return completion_tokens, t_first, t_last
|
||||
|
||||
|
||||
def scrape_metrics(base_url, timeout=10):
|
||||
"""Return dict of spec-decode counters from vLLM /metrics, if present."""
|
||||
keys = ("num_accepted_tokens", "num_draft_tokens", "num_drafts",
|
||||
"accepted_tokens", "draft_tokens")
|
||||
out = {}
|
||||
try:
|
||||
with urllib.request.urlopen(base_url.rstrip("/") + "/metrics", timeout=timeout) as r:
|
||||
for line in r.read().decode("utf-8", "replace").splitlines():
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
if "spec_decode" in line or "speculat" in line:
|
||||
name, _, val = line.partition(" ")
|
||||
try:
|
||||
out[name] = out.get(name, 0.0) + float(val)
|
||||
except ValueError:
|
||||
pass
|
||||
except (urllib.error.URLError, OSError):
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://127.0.0.1:8000")
|
||||
ap.add_argument("--model", required=True)
|
||||
ap.add_argument("--max-tokens", type=int, default=256)
|
||||
ap.add_argument("--runs", type=int, default=5)
|
||||
ap.add_argument("--warmup", type=int, default=1)
|
||||
ap.add_argument("--timeout", type=float, default=600)
|
||||
ap.add_argument("--label", default="")
|
||||
ap.add_argument("--prompt", default=PROMPT)
|
||||
args = ap.parse_args()
|
||||
|
||||
m_before = scrape_metrics(args.base_url)
|
||||
for _ in range(args.warmup):
|
||||
try:
|
||||
post_stream(args.base_url, args.model, args.prompt, 32, args.timeout)
|
||||
except Exception as e:
|
||||
print(f"warmup failed: {e}", file=sys.stderr); sys.exit(2)
|
||||
|
||||
decode_tps, ttfts, e2e_tps = [], [], []
|
||||
for i in range(args.runs):
|
||||
t0 = time.perf_counter()
|
||||
toks, tf, tl = post_stream(args.base_url, args.model, args.prompt, args.max_tokens, args.timeout)
|
||||
t1 = time.perf_counter()
|
||||
if not toks or tf is None or tl is None or tl <= tf:
|
||||
print(f" run {i}: degenerate (toks={toks})", file=sys.stderr); continue
|
||||
dec = (toks - 1) / (tl - tf)
|
||||
decode_tps.append(dec); ttfts.append((tf - t0) * 1000); e2e_tps.append(toks / (t1 - t0))
|
||||
print(f" run {i}: {toks} tok decode={dec:6.1f} tok/s ttft={ (tf-t0)*1000:6.0f} ms")
|
||||
m_after = scrape_metrics(args.base_url)
|
||||
|
||||
if not decode_tps:
|
||||
print("no successful runs", file=sys.stderr); sys.exit(1)
|
||||
print(f"\n=== {args.label or args.model} ===")
|
||||
print(f"decode tok/s : median {statistics.median(decode_tps):.1f} "
|
||||
f"mean {statistics.mean(decode_tps):.1f} min {min(decode_tps):.1f} max {max(decode_tps):.1f}")
|
||||
print(f"ttft ms : median {statistics.median(ttfts):.0f}")
|
||||
print(f"e2e tok/s : median {statistics.median(e2e_tps):.1f}")
|
||||
|
||||
# spec-decode acceptance length, if counters moved
|
||||
def delta(k):
|
||||
return m_after.get(k, 0.0) - m_before.get(k, 0.0)
|
||||
acc = next((delta(k) for k in m_after if "accepted" in k), 0.0)
|
||||
drafts = next((delta(k) for k in m_after if "num_drafts" in k or ("draft" in k and "tokens" not in k)), 0.0)
|
||||
dtoks = next((delta(k) for k in m_after if "draft_tokens" in k or "num_draft_tokens" in k), 0.0)
|
||||
if acc or dtoks:
|
||||
al = (acc / drafts) if drafts else float("nan")
|
||||
rate = (acc / dtoks) if dtoks else float("nan")
|
||||
print(f"spec accept : +{acc:.0f} accepted, +{dtoks:.0f} drafted, "
|
||||
f"mean accept len ~{al:.2f}, accept rate ~{rate:.1%}")
|
||||
else:
|
||||
print("spec accept : (no spec-decode counters — baseline/no drafter)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real-world DFlash/MTP acceptance bench on the actual Hermes agent workload.
|
||||
|
||||
Reconstructs real conversation contexts from ~/.hermes/state.db and has the loaded
|
||||
server regenerate the *next assistant turn* (so the model that produced the history is
|
||||
irrelevant — only the realistic context matters). Measures mean acceptance length
|
||||
(authoritative vLLM /metrics deltas) + decode tok/s, aggregated over N real turns.
|
||||
Privacy: runs entirely on the box; prints only metrics, not conversation content.
|
||||
"""
|
||||
import argparse, json, sqlite3, sys, time, urllib.request, urllib.error
|
||||
|
||||
DB = "/home/ent/.hermes/state.db"
|
||||
|
||||
|
||||
def scrape(base):
|
||||
out = {"acc": 0.0, "drafts": 0.0, "dtoks": 0.0}
|
||||
try:
|
||||
txt = urllib.request.urlopen(base.rstrip("/") + "/metrics", timeout=10).read().decode("utf-8", "replace")
|
||||
except Exception:
|
||||
return out
|
||||
for ln in txt.splitlines():
|
||||
if ln.startswith("#"):
|
||||
continue
|
||||
v = ln.split()[-1] if ln.split() else "0"
|
||||
try:
|
||||
val = float(v)
|
||||
except ValueError:
|
||||
continue
|
||||
if "spec_decode_num_accepted_tokens_total" in ln:
|
||||
out["acc"] += val
|
||||
elif "spec_decode_num_drafts_total" in ln:
|
||||
out["drafts"] += val
|
||||
elif "spec_decode_num_draft_tokens_total" in ln:
|
||||
out["dtoks"] += val
|
||||
return out
|
||||
|
||||
|
||||
def build_messages(cur, session_id, system_prompt, char_budget=24000):
|
||||
rows = list(cur.execute(
|
||||
"SELECT role, content, tool_calls, tool_call_id, tool_name FROM messages "
|
||||
"WHERE session_id=? ORDER BY id", (session_id,)))
|
||||
# find the LAST assistant turn -> generate it; prompt = everything before it
|
||||
last_asst = None
|
||||
for i, r in enumerate(rows):
|
||||
if r[0] == "assistant":
|
||||
last_asst = i
|
||||
if last_asst is None or last_asst == 0:
|
||||
return None
|
||||
pre = rows[:last_asst]
|
||||
msgs = []
|
||||
for role, content, tool_calls, tool_call_id, tool_name in pre:
|
||||
content = content or ""
|
||||
if role == "user":
|
||||
msgs.append({"role": "user", "content": content})
|
||||
elif role == "assistant":
|
||||
m = {"role": "assistant", "content": content}
|
||||
if tool_calls:
|
||||
try:
|
||||
tc = json.loads(tool_calls)
|
||||
if isinstance(tc, list) and tc:
|
||||
m["tool_calls"] = tc
|
||||
if not content:
|
||||
m["content"] = ""
|
||||
except Exception:
|
||||
pass
|
||||
msgs.append(m)
|
||||
elif role == "tool":
|
||||
msgs.append({"role": "tool", "content": content,
|
||||
"tool_call_id": tool_call_id or "call_0"})
|
||||
# skip session_meta
|
||||
if not msgs:
|
||||
return None
|
||||
# truncate oldest non-system messages to fit budget
|
||||
sys_msg = [{"role": "system", "content": system_prompt}] if system_prompt else []
|
||||
while sum(len(json.dumps(m)) for m in msgs) > char_budget and len(msgs) > 1:
|
||||
msgs.pop(0)
|
||||
return sys_msg + msgs
|
||||
|
||||
|
||||
def stream_chat(base, model, messages, max_tokens, timeout):
|
||||
body = json.dumps({"model": model, "messages": messages, "max_tokens": max_tokens,
|
||||
"temperature": 0.0, "stream": True,
|
||||
"stream_options": {"include_usage": True}}).encode()
|
||||
req = urllib.request.Request(base.rstrip("/") + "/v1/chat/completions", data=body,
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer x"})
|
||||
t_first = t_last = None
|
||||
toks = 0
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
for raw in r:
|
||||
ln = raw.decode("utf-8", "replace").strip()
|
||||
if not ln.startswith("data:"):
|
||||
continue
|
||||
d = ln[5:].strip()
|
||||
if d == "[DONE]":
|
||||
break
|
||||
try:
|
||||
o = json.loads(d)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
u = o.get("usage")
|
||||
if u and u.get("completion_tokens"):
|
||||
toks = u["completion_tokens"]
|
||||
ch = o.get("choices") or []
|
||||
if ch and (ch[0].get("delta") or {}).get("content"):
|
||||
now = time.perf_counter()
|
||||
if t_first is None:
|
||||
t_first = now
|
||||
t_last = now
|
||||
return toks, t_first, t_last
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://127.0.0.1:8000")
|
||||
ap.add_argument("--model", default="qwen")
|
||||
ap.add_argument("--max-tokens", type=int, default=200)
|
||||
ap.add_argument("--n-samples", type=int, default=10)
|
||||
ap.add_argument("--min-msgs", type=int, default=6)
|
||||
ap.add_argument("--timeout", type=float, default=300)
|
||||
ap.add_argument("--label", default="HERMES real")
|
||||
args = ap.parse_args()
|
||||
|
||||
db = sqlite3.connect(DB)
|
||||
cur = db.cursor()
|
||||
sess = list(cur.execute(
|
||||
"SELECT s.id, s.system_prompt FROM sessions s "
|
||||
"WHERE s.message_count >= ? ORDER BY s.started_at DESC LIMIT 40", (args.min_msgs,)))
|
||||
|
||||
m0 = scrape(args.base_url)
|
||||
decode_tps, used = [], 0
|
||||
for sid, sysp in sess:
|
||||
if used >= args.n_samples:
|
||||
break
|
||||
try:
|
||||
msgs = build_messages(cur, sid, sysp)
|
||||
except Exception:
|
||||
continue
|
||||
if not msgs:
|
||||
continue
|
||||
try:
|
||||
toks, tf, tl = stream_chat(args.base_url, args.model, msgs, args.max_tokens, args.timeout)
|
||||
except (urllib.error.HTTPError, urllib.error.URLError, OSError) as e:
|
||||
print(f" skip {sid[:24]}: {type(e).__name__}", file=sys.stderr)
|
||||
continue
|
||||
if toks and tf and tl and tl > tf:
|
||||
dec = (toks - 1) / (tl - tf)
|
||||
decode_tps.append(dec)
|
||||
used += 1
|
||||
print(f" sample {used}: {toks} tok decode={dec:5.1f} tok/s ({sid[:28]})")
|
||||
m1 = scrape(args.base_url)
|
||||
|
||||
dacc = m1["acc"] - m0["acc"]
|
||||
ddr = m1["drafts"] - m0["drafts"]
|
||||
ddt = m1["dtoks"] - m0["dtoks"]
|
||||
import statistics
|
||||
print(f"\n=== {args.label} (n={used} real turns) ===")
|
||||
if decode_tps:
|
||||
print(f"decode tok/s : median {statistics.median(decode_tps):.1f} mean {statistics.mean(decode_tps):.1f} "
|
||||
f"min {min(decode_tps):.1f} max {max(decode_tps):.1f}")
|
||||
if ddr > 0:
|
||||
print(f"accept length: {1 + dacc/ddr:.2f} (accepted {dacc:.0f} / drafts {ddr:.0f}; "
|
||||
f"draft tokens {ddt:.0f}; accept rate {dacc/ddt:.1%})")
|
||||
else:
|
||||
print("accept length: (no draft activity captured)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# Monitor SGLang container startup with an OOM auto-kill guard.
|
||||
# Breaks on: READY, ERROR (traceback), OOM-GUARD (avail mem too low), or timeout.
|
||||
set -u
|
||||
NAME="${1:-sglang-dflash}"
|
||||
MAX_ITERS="${2:-60}" # 60 * 20s = 20 min
|
||||
FLOOR_MB="${3:-4096}" # kill if available memory drops below this
|
||||
|
||||
for i in $(seq 1 "$MAX_ITERS"); do
|
||||
if ! docker ps --filter "name=$NAME" --format '{{.Names}}' | grep -q "$NAME"; then
|
||||
echo "STATE=EXITED iter=$i"
|
||||
echo "--- last log ---"; docker logs "$NAME" 2>&1 | tail -25
|
||||
exit 0
|
||||
fi
|
||||
AVAIL=$(free -m | awk '/^Mem:/{print $7}')
|
||||
if [ "$AVAIL" -lt "$FLOOR_MB" ]; then
|
||||
echo "STATE=OOM-GUARD iter=$i avail_mb=$AVAIL -> killing $NAME"
|
||||
docker kill "$NAME" >/dev/null 2>&1
|
||||
echo "--- last log ---"; docker logs "$NAME" 2>&1 | tail -25
|
||||
exit 0
|
||||
fi
|
||||
LOG=$(docker logs "$NAME" 2>&1)
|
||||
if echo "$LOG" | grep -qiE "server is fired up|Application startup complete|The server is ready"; then
|
||||
echo "STATE=READY iter=$i avail_mb=$AVAIL"
|
||||
echo "--- tail ---"; echo "$LOG" | tail -20
|
||||
exit 0
|
||||
fi
|
||||
if echo "$LOG" | grep -qE "Traceback \(most recent call last\)|CUDA out of memory|RuntimeError|AssertionError|ValueError|raise NotImplementedError"; then
|
||||
echo "STATE=ERROR iter=$i avail_mb=$AVAIL"
|
||||
echo "--- tail ---"; echo "$LOG" | tail -40
|
||||
exit 0
|
||||
fi
|
||||
LAST=$(echo "$LOG" | tail -1 | cut -c1-110)
|
||||
echo "iter=$i avail_mb=$AVAIL :: $LAST"
|
||||
sleep 20
|
||||
done
|
||||
echo "STATE=TIMEOUT after $MAX_ITERS iters"
|
||||
docker logs "$NAME" 2>&1 | tail -20
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
# Run the 4-workload acceptance/throughput bank on whichever server is up.
|
||||
# $1 = container name (for the authoritative server-log accept_len). Uses the SAME
|
||||
# prompts as the DFlash run so DFlash vs MTP is apples-to-apples.
|
||||
CN="${1:-vllm-mtp}"
|
||||
BU=http://127.0.0.1:8000; M=qwen
|
||||
acc_tail(){ docker logs "$CN" 2>&1 | grep "Mean acceptance length" | tail -1 \
|
||||
| sed -E "s/.*Mean acceptance length: ([0-9.]+).*Per-position acceptance rate: ([0-9., ]+), Avg.*/accept_len=\1 per-pos=[\2]/"; }
|
||||
run(){ L="$1"; P="$2"; echo "########## $L ##########"
|
||||
python3 ~/bench_decode.py --base-url $BU --model $M --max-tokens 256 --runs 4 --warmup 1 --label "$L" --prompt "$P" 2>&1 | grep -E "decode tok/s"
|
||||
echo " server $(acc_tail)"; }
|
||||
run "PROSE" "You are a careful writer. Write a long flowing essay about the history and philosophy of science, with no lists. Begin: "
|
||||
run "CODE" "Write a complete Python implementation of a binary search tree class with insert, search, delete, and inorder traversal. Include docstrings and type hints. Begin:\n\nclass BSTNode:"
|
||||
run "COUNTING" "Write all the whole numbers from 1 to 400, separated by commas, with no other text. Begin: 1, 2, 3, "
|
||||
echo "########## HERMES (real agent turns) ##########"
|
||||
python3 ~/hermes_bench.py --base-url $BU --model $M --label "HERMES" --n-samples 10 --max-tokens 200 2>&1 | grep -E "decode tok/s|accept length"
|
||||
Reference in New Issue
Block a user