qwen3.5-122B-A10B on DGX Spark: vLLM + DFlash + dense-bandwidth stack, one-shot installer

2026-06-24 13:02:35 +10:00
commit 60bf1b7b02
21 changed files with 2353 additions and 0 deletions
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""Inspect the Intel INT4 AutoRound checkpoint to decide if albond's hybrid
+INT4+FP8 build will work: it swaps BF16 *dense* (non-expert) tensors for FP8 by
+name. If AutoRound quantized the dense linears to INT4 (.qweight), the swap is a
+near no-op. We need attention/shared_expert dense weights stored as BF16 (.weight).
+
+Also pulls the FP8 repo's index to confirm it exists and that names line up.
+"""
+import json
+import sys
+from collections import Counter
+from pathlib import Path
+
+INT4_DIR = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(
+    "/home/ent/.cache/huggingface/hub/models--Intel--Qwen3.5-122B-A10B-int4-AutoRound/"
+    "snapshots/3045d02bb737effc4581da91bddbad3be02934e4")
+FP8_REPO = sys.argv[2] if len(sys.argv) > 2 else "Qwen/Qwen3.5-122B-A10B-FP8"
+
+idx = json.loads((INT4_DIR / "model.safetensors.index.json").read_text())
+wm = idx["weight_map"]
+names = list(wm.keys())
+
+# Non-expert = no ".experts." (routed experts stay INT4 in the hybrid)
+nonexp = [n for n in names if ".experts." not in n]
+
+def bucket(n):
+    if ".experts." in n:
+        return "ROUTED-EXPERT"
+    if "shared_expert" in n:
+        return "shared_expert"
+    if "self_attn" in n or "linear_attn" in n or ".attn" in n:
+        return "attention"
+    if "embed_tokens" in n or "lm_head" in n:
+        return "embed/head"
+    if "mtp" in n:
+        return "mtp"
+    return "other"
+
+print(f"=== Intel INT4 checkpoint: {INT4_DIR.name} ===")
+print(f"total tensors: {len(names)}  non-expert: {len(nonexp)}")
+
+# Suffix histogram tells us quant scheme: .qweight/.scales/.qzeros => INT4; .weight => dense
+suffix = Counter(n.rsplit(".", 1)[-1] for n in nonexp)
+print("\nnon-expert tensor SUFFIX histogram (qweight/scales/qzeros = INT4-packed; weight = dense):")
+for s, c in suffix.most_common():
+    print(f"  .{s:20s} {c}")
+
+# For each functional group, does it have .weight (BF16 dense) or .qweight (INT4)?
+print("\nper-group quant scheme (sample names):")
+groups = {}
+for n in nonexp:
+    g = bucket(n)
+    groups.setdefault(g, {"weight": 0, "qweight": 0, "scale": 0, "other": 0, "ex": None})
+    suf = n.rsplit(".", 1)[-1]
+    if suf == "weight":
+        groups[g]["weight"] += 1
+    elif suf == "qweight":
+        groups[g]["qweight"] += 1
+    elif "scale" in suf or suf in ("qzeros",):
+        groups[g]["scale"] += 1
+    else:
+        groups[g]["other"] += 1
+    if groups[g]["ex"] is None and suf in ("weight", "qweight"):
+        groups[g]["ex"] = n
+for g, d in sorted(groups.items()):
+    scheme = "INT4(.qweight)" if d["qweight"] else ("DENSE(.weight)" if d["weight"] else "?")
+    print(f"  {g:16s} weight={d['weight']:4d} qweight={d['qweight']:4d} scale={d['scale']:4d}  -> {scheme}")
+    print(f"      e.g. {d['ex']}")
+
+# Dtypes of a few non-expert .weight tensors (open the shard header only).
+print("\ndtypes of sample non-expert '.weight' tensors (FP8 swap needs BF16 here):")
+from safetensors import safe_open  # noqa: E402
+sample = [n for n in nonexp if n.endswith(".weight")
+          and ("self_attn" in n or "shared_expert" in n or "embed" in n or "lm_head" in n)]
+seen_shards = {}
+shown = 0
+for n in sample:
+    shard = wm[n]
+    f = seen_shards.get(shard)
+    if f is None:
+        f = safe_open(str(INT4_DIR / shard), framework="pt")
+        seen_shards[shard] = f
+    try:
+        t = f.get_slice(n)
+        print(f"  {n:60s} {t.get_dtype()} {tuple(t.get_shape())}")
+    except Exception as e:
+        print(f"  {n:60s} <err {e}>")
+    shown += 1
+    if shown >= 12:
+        break
+
+# FP8 repo: confirm exists + list its non-expert names/dtypes for name-match sanity.
+print(f"\n=== FP8 repo manifest: {FP8_REPO} ===")
+try:
+    from huggingface_hub import hf_hub_download
+    p = hf_hub_download(FP8_REPO, "model.safetensors.index.json")
+    fidx = json.loads(Path(p).read_text())
+    fwm = fidx["weight_map"]
+    fnon = [n for n in fwm if ".experts." not in n]
+    print(f"FP8 total tensors: {len(fwm)}  non-expert: {len(fnon)}")
+    fsuf = Counter(n.rsplit('.', 1)[-1] for n in fnon)
+    print("FP8 non-expert suffix histogram:")
+    for s, c in fsuf.most_common(10):
+        print(f"  .{s:20s} {c}")
+    # how many FP8 non-expert .weight names also exist in INT4 as .weight?
+    int4_weight = {n for n in nonexp if n.endswith('.weight')}
+    fp8_weight = {n for n in fnon if n.endswith('.weight')}
+    match = int4_weight & fp8_weight
+    print(f"\nname overlap (FP8 '.weight' that also exist as '.weight' in INT4): {len(match)} / {len(fp8_weight)} FP8 weights")
+    only_fp8 = sorted(fp8_weight - int4_weight)[:8]
+    print(f"FP8 '.weight' NOT present as '.weight' in INT4 (would not swap): {len(fp8_weight - int4_weight)}")
+    for n in only_fp8:
+        print(f"    {n}  (INT4 has: {'qweight' if n[:-7]+'.qweight' in set(names) else 'MISSING'})")
+except Exception as e:
+    print(f"<FP8 repo fetch failed: {e}>")