From e8115e7aa69c4a9df49dd34f02d53d838cc2094b Mon Sep 17 00:00:00 2001 From: TBNilles Date: Sun, 7 Jun 2026 14:50:10 -0400 Subject: [PATCH] feat(model-manager): add CivitAI browse view + fix civitai.red / 401 - New "Browse CivitAI" view: thumbnail grid with search, type/sort/period filters and NSFW toggle; click a model card to download it (per-card version picker for multi-version models). Cursor + page based "Load more". - Backend: /api/civitai/search and /api/civitai/download endpoints; new civitai_search() catalog helper. - Fix 401 on paste: recognize the civitai.red mirror (and any civitai.* host), normalize API calls to civitai.com, and always resolve the model-version so type + filename are auto-detected for every CivitAI URL. Co-Authored-By: Claude Opus 4.8 --- README.md | 7 +- model-manager/app/main.py | 33 +++++ model-manager/app/registries.py | 179 ++++++++++++++++++++++------ model-manager/app/static/app.js | 157 ++++++++++++++++++++++++ model-manager/app/static/index.html | 50 ++++++++ model-manager/app/static/style.css | 42 +++++++ 6 files changed, 427 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index b960c1a..d28a619 100644 --- a/README.md +++ b/README.md @@ -235,11 +235,14 @@ SparkyUI includes a **StabilityMatrix-style Model Manager** - a lightweight Fast **Access:** `http://:8189` **Features:** +- **Browse CivitAI** - search the CivitAI catalog in a thumbnail grid (filter by type, + sort, period, NSFW toggle) and **click a model to download it** - no URL pasting needed. + Multi-version models get a version picker on the card. - **Installed Models** - browse what's on disk, grouped by type, with size and delete actions - **Add / Download** - paste a download URL and pick a type; live progress bars - **Direct URLs** - any direct download link - - **CivitAI** - paste a model page link (`civitai.com/models/...`) or an - `api/download/models/...` link; the type and filename are auto-detected + - **CivitAI** - paste a model page link (`civitai.com/models/...`, the `civitai.red` + mirror, or an `api/download/models/...` link); the type and filename are auto-detected - **HuggingFace** - paste a `resolve` URL (works with gated repos via your token) - **Settings** - store your **CivitAI API key** and **HuggingFace token** persistently (saved to a SQLite DB under `./sparkyui-data`, never committed to git) diff --git a/model-manager/app/main.py b/model-manager/app/main.py index ae41c46..0953dd7 100644 --- a/model-manager/app/main.py +++ b/model-manager/app/main.py @@ -169,6 +169,39 @@ async def start_download(body: DownloadIn) -> dict: return row or {"id": download_id} +# ---- CivitAI catalog browse ---------------------------------------------- + +@app.get("/api/civitai/search") +async def civitai_search( + query: Optional[str] = None, + types: Optional[str] = None, # comma-separated CivitAI types + sort: Optional[str] = "Most Downloaded", + period: Optional[str] = "AllTime", + nsfw: bool = False, + page: Optional[int] = None, + cursor: Optional[str] = None, +) -> dict: + if not db.get_setting("civitai_api_key"): + raise HTTPException(400, "Set your CivitAI API key in Settings first.") + type_list = [t for t in (types or "").split(",") if t] or None + try: + return await registries.civitai_search( + query=query, types=type_list, sort=sort, period=period, + nsfw=nsfw, page=page, cursor=cursor) + except Exception as exc: # noqa: BLE001 - surface API errors to the UI + raise HTTPException(502, f"CivitAI search failed: {exc}") + + +class CivitaiDownloadIn(BaseModel): + version_id: int + + +@app.post("/api/civitai/download") +async def civitai_download(body: CivitaiDownloadIn) -> dict: + url = f"https://civitai.com/api/download/models/{body.version_id}" + return await start_download(DownloadIn(url=url)) + + @app.delete("/api/downloads/{download_id}") def delete_download(download_id: int) -> dict: row = db.get_download(download_id) diff --git a/model-manager/app/registries.py b/model-manager/app/registries.py index a6d37b2..5254fe6 100644 --- a/model-manager/app/registries.py +++ b/model-manager/app/registries.py @@ -1,7 +1,8 @@ -"""Resolve a user-supplied URL into a concrete download (direct URL + auth + metadata). +"""Resolve a user-supplied URL into a concrete download (direct URL + auth + metadata) +and provide CivitAI catalog search. -Supports direct URLs, CivitAI (model/version pages or api download links), and -HuggingFace `resolve` URLs. API keys are read from the settings table. +Supports direct URLs, CivitAI (model/version pages, civitai.red mirror, or api +download links), and HuggingFace `resolve` URLs. API keys are read from settings. """ from __future__ import annotations @@ -14,6 +15,8 @@ import httpx from . import db +CIVITAI_API = "https://civitai.com/api/v1" + # CivitAI model `type` -> our model-type key. CIVITAI_TYPE_MAP = { "Checkpoint": "checkpoint", @@ -28,6 +31,7 @@ CIVITAI_TYPE_MAP = { "MotionModule": "other", "Poses": "other", "Wildcards": "other", + "Workflows": "other", "Other": "other", } @@ -41,9 +45,14 @@ class Resolved: model_type: Optional[str] = None # suggested type if the registry tells us +def _is_civitai_host(host: str) -> bool: + # Matches civitai.com, civitai.red, and any civitai.* mirror. + return "civitai" in host + + def detect_source(url: str) -> str: host = (urlparse(url).hostname or "").lower() - if "civitai.com" in host: + if _is_civitai_host(host): return "civitai" if "huggingface.co" in host or "hf.co" in host: return "huggingface" @@ -56,6 +65,14 @@ def _filename_from_url(url: str) -> Optional[str]: return name or None +def civitai_headers() -> dict[str, str]: + headers: dict[str, str] = {} + key = db.get_setting("civitai_api_key") + if key: + headers["Authorization"] = f"Bearer {key}" + return headers + + async def resolve(url: str) -> Resolved: source = detect_source(url) if source == "civitai": @@ -83,49 +100,50 @@ _CIVITAI_VERSION_RE = re.compile(r"/api/download/models/(\d+)") _CIVITAI_MODEL_RE = re.compile(r"/models/(\d+)") -async def _resolve_civitai(url: str) -> Resolved: - headers: dict[str, str] = {} - api_key = db.get_setting("civitai_api_key") - if api_key: - headers["Authorization"] = f"Bearer {api_key}" +async def _civitai_get(path: str) -> dict: + async with httpx.AsyncClient(timeout=30, headers=civitai_headers()) as client: + resp = await client.get(f"{CIVITAI_API}{path}") + resp.raise_for_status() + return resp.json() - # Case 1: already a direct api/download URL -> use as-is. - if _CIVITAI_VERSION_RE.search(urlparse(url).path): - return Resolved(download_url=url, source="civitai", headers=headers, - filename=_filename_from_url(url)) - # Case 2: a model page URL. Find the version id (explicit query or first version). +async def _civitai_version_id(url: str) -> str: + """Extract or look up the model-version id from any CivitAI URL form.""" parsed = urlparse(url) - qs = parse_qs(parsed.query) - version_id: Optional[str] = None - if "modelVersionId" in qs: - version_id = qs["modelVersionId"][0] - if version_id is None: - m = _CIVITAI_MODEL_RE.search(parsed.path) - if not m: - # Can't understand it; fall back to treating it as a direct link. - return Resolved(download_url=url, source="civitai", headers=headers, - filename=_filename_from_url(url)) - model_id = m.group(1) - async with httpx.AsyncClient(timeout=30, headers=headers) as client: - resp = await client.get(f"https://civitai.com/api/v1/models/{model_id}") - resp.raise_for_status() - data = resp.json() + # api/download/models/{versionId} + m = _CIVITAI_VERSION_RE.search(parsed.path) + if m: + return m.group(1) + + # ...?modelVersionId=... + qs = parse_qs(parsed.query) + if "modelVersionId" in qs: + return qs["modelVersionId"][0] + + # model page: /models/{id} -> first version + m = _CIVITAI_MODEL_RE.search(parsed.path) + if m: + data = await _civitai_get(f"/models/{m.group(1)}") versions = data.get("modelVersions") or [] if not versions: raise ValueError("CivitAI model has no downloadable versions") - version_id = str(versions[0]["id"]) + return str(versions[0]["id"]) - # Resolve the version to a concrete file + type. - async with httpx.AsyncClient(timeout=30, headers=headers) as client: - resp = await client.get( - f"https://civitai.com/api/v1/model-versions/{version_id}") - resp.raise_for_status() - version = resp.json() + raise ValueError("Unrecognized CivitAI URL") + + +async def _resolve_civitai(url: str) -> Resolved: + """Resolve any CivitAI URL to a concrete file via the model-versions API. + + Always looks the version up so we get the real filename and model type + (works for civitai.com and the civitai.red mirror alike). + """ + headers = civitai_headers() + version_id = await _civitai_version_id(url) + version = await _civitai_get(f"/model-versions/{version_id}") files = version.get("files") or [] - # Prefer the primary file, else the first. chosen = next((f for f in files if f.get("primary")), files[0] if files else None) if not chosen: raise ValueError("CivitAI version has no files") @@ -133,10 +151,93 @@ async def _resolve_civitai(url: str) -> Resolved: civ_type = (version.get("model") or {}).get("type") or "Other" model_type = CIVITAI_TYPE_MAP.get(civ_type, "other") + download_url = chosen.get("downloadUrl") or \ + f"https://civitai.com/api/download/models/{version_id}" + return Resolved( - download_url=chosen["downloadUrl"], + download_url=download_url, source="civitai", headers=headers, - filename=chosen.get("name") or _filename_from_url(chosen["downloadUrl"]), + filename=chosen.get("name") or _filename_from_url(download_url), model_type=model_type, ) + + +# ---- catalog search ------------------------------------------------------- + +def _pick_thumbnail(version: dict) -> Optional[dict]: + """Return {url, type} of a representative preview for a model version.""" + images = version.get("images") or [] + for im in images: + if im.get("type") == "image": + return {"url": im.get("url"), "type": "image"} + if images: + return {"url": images[0].get("url"), "type": images[0].get("type", "image")} + return None + + +def _to_card(item: dict) -> dict: + versions = item.get("modelVersions") or [] + v0 = versions[0] if versions else {} + thumb = _pick_thumbnail(v0) if v0 else None + stats = item.get("stats") or {} + return { + "id": item.get("id"), + "name": item.get("name"), + "type": item.get("type"), + "type_key": CIVITAI_TYPE_MAP.get(item.get("type") or "", "other"), + "nsfw": item.get("nsfw", False), + "creator": (item.get("creator") or {}).get("username"), + "downloads": stats.get("downloadCount"), + "thumbnail": thumb.get("url") if thumb else None, + "thumbnail_type": thumb.get("type") if thumb else None, + "primary_version_id": v0.get("id"), + "versions": [ + {"id": v.get("id"), "name": v.get("name")} for v in versions + ], + } + + +async def civitai_search(query: Optional[str] = None, + types: Optional[list[str]] = None, + sort: Optional[str] = None, + period: Optional[str] = None, + nsfw: Optional[bool] = None, + page: Optional[int] = None, + cursor: Optional[str] = None, + limit: int = 24) -> dict: + """Search the CivitAI model catalog, returning normalized cards + paging. + + CivitAI uses cursor paging whenever a `query` is supplied (page is ignored), + and page paging otherwise, so we support and return both. + """ + params: dict = {"limit": max(1, min(limit, 100))} + if query: + params["query"] = query + if types: + params["types"] = types # httpx serializes lists as repeated params + if sort: + params["sort"] = sort + if period: + params["period"] = period + if nsfw is not None: + params["nsfw"] = "true" if nsfw else "false" + if cursor: + params["cursor"] = cursor + elif page: + params["page"] = page + + async with httpx.AsyncClient(timeout=30, headers=civitai_headers()) as client: + resp = await client.get(f"{CIVITAI_API}/models", params=params) + resp.raise_for_status() + data = resp.json() + + items = [_to_card(it) for it in (data.get("items") or [])] + meta = data.get("metadata") or {} + return { + "items": items, + "nextPage": meta.get("nextPage"), + "nextCursor": meta.get("nextCursor"), + "currentPage": meta.get("currentPage"), + "totalPages": meta.get("totalPages"), + } diff --git a/model-manager/app/static/app.js b/model-manager/app/static/app.js index c84d55a..309c43e 100644 --- a/model-manager/app/static/app.js +++ b/model-manager/app/static/app.js @@ -38,6 +38,7 @@ function showView(name) { $$(".nav-item").forEach((b) => b.classList.toggle("active", b.dataset.view === name)); $$(".view").forEach((v) => v.classList.toggle("active", v.id === `view-${name}`)); if (name === "installed") loadModels(); + if (name === "browse" && !browseState.loaded) runSearch(true); if (name === "download") { loadDownloads(); startDownloadsPolling(); } else stopDownloadsPolling(); if (name === "settings") loadSettings(); @@ -243,6 +244,162 @@ function stopDownloadsPolling() { if (downloadsTimer) { clearInterval(downloadsTimer); downloadsTimer = null; } } +// ---- browse civitai ------------------------------------------------------- + +const browseState = { loaded: false, page: null, cursor: null, busy: false }; + +function browseParams() { + const p = new URLSearchParams(); + const q = $("#bSearch").value.trim(); + if (q) p.set("query", q); + if ($("#bType").value) p.set("types", $("#bType").value); + p.set("sort", $("#bSort").value); + p.set("period", $("#bPeriod").value); + p.set("nsfw", $("#bNsfw").checked ? "true" : "false"); + return p; +} + +async function runSearch(reset) { + if (browseState.busy) return; + browseState.busy = true; + browseState.loaded = true; + const grid = $("#browseGrid"); + const moreBtn = $("#bMore"); + const p = browseParams(); + + if (reset) { + browseState.page = null; + browseState.cursor = null; + grid.innerHTML = `

Searching…

`; + } else { + if (browseState.cursor) p.set("cursor", browseState.cursor); + else if (browseState.page) p.set("page", browseState.page); + moreBtn.textContent = "Loading…"; + } + + try { + const data = await api(`/api/civitai/search?${p.toString()}`); + if (reset) grid.innerHTML = ""; + if (!data.items.length && reset) { + grid.innerHTML = `

No models found.

`; + } else { + for (const m of data.items) grid.appendChild(renderBrowseCard(m)); + } + // Set up "load more": prefer cursor, fall back to next page. + browseState.cursor = data.nextCursor || null; + browseState.page = data.nextPage || null; + const hasMore = Boolean(browseState.cursor || browseState.page); + moreBtn.style.display = hasMore ? "" : "none"; + moreBtn.textContent = "Load more"; + } catch (err) { + if (reset) grid.innerHTML = `

Error: ${err.message}

`; + else toast($("#browseToast"), err.message, true); + moreBtn.textContent = "Load more"; + } finally { + browseState.busy = false; + } +} + +function renderBrowseCard(m) { + const card = document.createElement("div"); + card.className = "browse-card"; + + const thumb = document.createElement("div"); + thumb.className = "thumb"; + if (m.thumbnail && m.thumbnail_type === "image") { + thumb.style.backgroundImage = `url("${m.thumbnail}")`; + } + if (m.type) { + const tag = document.createElement("span"); + tag.className = "type-tag"; + tag.textContent = m.type; + thumb.appendChild(tag); + } + if (m.nsfw) { + const n = document.createElement("span"); + n.className = "nsfw-tag"; + n.textContent = "NSFW"; + thumb.appendChild(n); + } + + const body = document.createElement("div"); + body.className = "body"; + const name = document.createElement("div"); + name.className = "b-name"; + name.textContent = m.name || "(untitled)"; + const meta = document.createElement("div"); + meta.className = "b-meta"; + meta.textContent = `${m.creator || "?"} · ${fmtNum(m.downloads)} downloads`; + + const foot = document.createElement("div"); + foot.className = "b-foot"; + + // Version selector when there's more than one. + let versionSel = null; + if (m.versions && m.versions.length > 1) { + versionSel = document.createElement("select"); + for (const v of m.versions) { + const o = document.createElement("option"); + o.value = v.id; + o.textContent = v.name || `v${v.id}`; + versionSel.appendChild(o); + } + foot.appendChild(versionSel); + } + + const btn = document.createElement("button"); + btn.className = "btn primary"; + btn.textContent = "Download"; + btn.addEventListener("click", () => { + const vid = versionSel ? Number(versionSel.value) : m.primary_version_id; + downloadVersion(vid, btn); + }); + foot.appendChild(btn); + + body.append(name, meta, foot); + card.append(thumb, body); + return card; +} + +async function downloadVersion(versionId, btn) { + if (!versionId) { toast($("#browseToast"), "No version available", true); return; } + btn.disabled = true; + const original = btn.textContent; + btn.textContent = "Queued ✓"; + try { + await api("/api/civitai/download", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ version_id: versionId }), + }); + toast($("#browseToast"), "Download started — see Add / Download", false); + setTimeout(() => { btn.disabled = false; btn.textContent = original; }, 2500); + } catch (err) { + toast($("#browseToast"), err.message, true); + btn.disabled = false; + btn.textContent = original; + } +} + +function fmtNum(n) { + if (n == null) return "?"; + if (n >= 1e6) return (n / 1e6).toFixed(1) + "M"; + if (n >= 1e3) return (n / 1e3).toFixed(1) + "k"; + return String(n); +} + +let toastTimer = null; +function toast(el, msg, isErr) { + el.textContent = msg; + el.className = `form-msg ${isErr ? "err" : "ok"}`; + if (toastTimer) clearTimeout(toastTimer); + toastTimer = setTimeout(() => { el.textContent = ""; }, 4000); +} + +$("#bGo").addEventListener("click", () => runSearch(true)); +$("#bSearch").addEventListener("keydown", (e) => { if (e.key === "Enter") runSearch(true); }); +$("#bMore").addEventListener("click", () => runSearch(false)); + // ---- settings ------------------------------------------------------------- async function loadSettings() { diff --git a/model-manager/app/static/index.html b/model-manager/app/static/index.html index ff2d696..ee84d21 100644 --- a/model-manager/app/static/index.html +++ b/model-manager/app/static/index.html @@ -20,6 +20,9 @@ + @@ -45,6 +48,53 @@ + +
+
+

Browse CivitAI

+
+ +
+
+ +
+ + + + + + +
+ +
+

Search the CivitAI catalog, then click Download on a model.

+
+
+ +
+
+

Add / Download Model

diff --git a/model-manager/app/static/style.css b/model-manager/app/static/style.css index 5bbe98f..d2e46ec 100644 --- a/model-manager/app/static/style.css +++ b/model-manager/app/static/style.css @@ -132,6 +132,48 @@ code { background: var(--bg-3); padding: 1px 5px; border-radius: 4px; font-size: .model-card .card-foot { display: flex; justify-content: space-between; align-items: center; margin-top: 4px; } .empty { color: var(--text-dim); } +/* ---- browse civitai ---- */ +.browse-controls { + display: flex; flex-wrap: wrap; gap: 10px; align-items: center; + margin-bottom: 20px; +} +.browse-controls input[type=search] { width: 280px; flex: 1; min-width: 200px; } +.browse-controls select { width: auto; } +.check { display: flex; align-items: center; gap: 6px; margin: 0; color: var(--text); white-space: nowrap; } +.check input { width: auto; } + +.browse-grid { + display: grid; gap: 14px; + grid-template-columns: repeat(auto-fill, minmax(210px, 1fr)); +} +.browse-card { + background: var(--bg-2); border: 1px solid var(--border); + border-radius: var(--radius); overflow: hidden; + display: flex; flex-direction: column; +} +.browse-card .thumb { + aspect-ratio: 3 / 4; background: var(--bg-3) center/cover no-repeat; + position: relative; +} +.browse-card .thumb .type-tag { + position: absolute; top: 8px; left: 8px; + background: rgba(12,16,32,.78); color: var(--text); + font-size: 11px; padding: 2px 8px; border-radius: 8px; +} +.browse-card .thumb .nsfw-tag { + position: absolute; top: 8px; right: 8px; + background: rgba(239,93,107,.85); color: #fff; + font-size: 10px; padding: 2px 7px; border-radius: 8px; font-weight: 600; +} +.browse-card .body { padding: 10px 12px; display: flex; flex-direction: column; gap: 6px; flex: 1; } +.browse-card .b-name { font-weight: 600; font-size: 13px; line-height: 1.25; + display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; } +.browse-card .b-meta { color: var(--text-dim); font-size: 11px; } +.browse-card .b-foot { margin-top: auto; display: flex; gap: 6px; align-items: center; } +.browse-card select { font-size: 12px; padding: 5px 6px; } +.browse-card .btn { padding: 6px 12px; font-size: 12px; flex: 1; } +.browse-more { display: flex; justify-content: center; margin: 22px 0; } + /* ---- downloads ---- */ .downloads-container { display: flex; flex-direction: column; gap: 10px; max-width: 860px; } .dl-row {