feat: support huggingface snapshot downloader

fixed: An issue where JS did not properly handle model download errors.
fixed: better security message for model downloading
This commit is contained in:
Dr.Lt.Data
2025-02-10 02:24:08 +09:00
parent 411c0633a3
commit 559c011420
8 changed files with 146 additions and 28 deletions

View File

@@ -3,6 +3,11 @@ from urllib.parse import urlparse
import urllib
import sys
import logging
import requests
from huggingface_hub import HfApi
from tqdm.auto import tqdm
aria2 = os.getenv('COMFYUI_MANAGER_ARIA2_SERVER')
HF_ENDPOINT = os.getenv('HF_ENDPOINT')
@@ -117,3 +122,37 @@ def download_url_with_agent(url, save_path):
print("Installation was successful.")
return True
# NOTE: snapshot_download doesn't provide file size tqdm.
def download_repo_in_bytes(repo_id, local_dir):
api = HfApi()
repo_info = api.repo_info(repo_id=repo_id, files_metadata=True)
os.makedirs(local_dir, exist_ok=True)
total_size = 0
for file_info in repo_info.siblings:
if file_info.size is not None:
total_size += file_info.size
pbar = tqdm(total=total_size, unit="B", unit_scale=True, desc="Downloading")
for file_info in repo_info.siblings:
out_path = os.path.join(local_dir, file_info.rfilename)
os.makedirs(os.path.dirname(out_path), exist_ok=True)
if file_info.size is None:
continue
download_url = f"https://huggingface.co/{repo_id}/resolve/main/{file_info.rfilename}"
with requests.get(download_url, stream=True) as r, open(out_path, "wb") as f:
r.raise_for_status()
for chunk in r.iter_content(chunk_size=65536):
if chunk:
f.write(chunk)
pbar.update(len(chunk))
pbar.close()