TBNilles/ComfyUI-Manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1915 from Comfy-Org/feat/implement-batch-tracking-clean

Browse Source 
[feat] Implement comprehensive batch tracking and OpenAPI-driven data models
This commit is contained in:
Christian Byrne
2025-06-21 19:46:23 -07:00
committed by GitHub
parent 7e51286313 a66f86d4af
commit cb0fa5829d
15 changed files with 3781 additions and 1862 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
comfyui_manager/glob/CLAUDE.md Normal file
View File

@@ -0,0 +1,11 @@
- Anytime you make a change to the data being sent or received, you should follow this process:
1. Adjust the openapi.yaml file first
2. Verify the syntax of the openapi.yaml file using `yaml.safe_load`
3. Regenerate the types following the instructions in the `data_models/README.md` file
4. Verify the new data model is generated
5. Verify the syntax of the generated types files
6. Run formatting and linting on the generated types files
7. Adjust the `__init__.py` files in the `data_models` directory to match/export the new data model
8. Only then, make the changes to the rest of the codebase
9. Run the CI tests to verify that the changes are working
- The comfyui_manager is a python package that is used to manage the comfyui server. There are two sub-packages `glob` and `legacy`. These represent the current version (`glob`) and the previous version (`legacy`), not including common utilities and data models. When developing, we work in the `glob` package. You can ignore the `legacy` package entirely, unless you have a very good reason to research how things were done in the legacy or prior major versions of the package. But in those cases, you should just look for the sake of knowledge or reflection, not for changing code (unless explicitly asked to do so).

54
comfyui_manager/glob/constants.py Normal file
View File

@@ -0,0 +1,54 @@
SECURITY_MESSAGE_MIDDLE_OR_BELOW = "ERROR: To use this action, a security_level of `middle or below` is required. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy"
SECURITY_MESSAGE_NORMAL_MINUS = "ERROR: To use this feature, you must either set '--listen' to a local IP and set the security level to 'normal-' or lower, or set the security level to 'middle' or 'weak'. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy"
SECURITY_MESSAGE_GENERAL = "ERROR: This installation is not allowed in this security_level. Please contact the administrator.\nReference: https://github.com/ltdrdata/ComfyUI-Manager#security-policy"
SECURITY_MESSAGE_NORMAL_MINUS_MODEL = "ERROR: Downloading models that are not in '.safetensors' format is only allowed for models registered in the 'default' channel at this security level. If you want to download this model, set the security level to 'normal-' or lower."
def is_loopback(address):
import ipaddress
try:
return ipaddress.ip_address(address).is_loopback
except ValueError:
return False
model_dir_name_map = {
"checkpoints": "checkpoints",
"checkpoint": "checkpoints",
"unclip": "checkpoints",
"text_encoders": "text_encoders",
"clip": "text_encoders",
"vae": "vae",
"lora": "loras",
"t2i-adapter": "controlnet",
"t2i-style": "controlnet",
"controlnet": "controlnet",
"clip_vision": "clip_vision",
"gligen": "gligen",
"upscale": "upscale_models",
"embedding": "embeddings",
"embeddings": "embeddings",
"unet": "diffusion_models",
"diffusion_model": "diffusion_models",
}
# List of all model directory names used for checking installed models
MODEL_DIR_NAMES = [
"checkpoints",
"loras",
"vae",
"text_encoders",
"diffusion_models",
"clip_vision",
"embeddings",
"diffusers",
"vae_approx",
"controlnet",
"gligen",
"upscale_models",
"hypernetworks",
"photomaker",
"classifiers",
]

2761
comfyui_manager/glob/manager_server.py
View File

File diff suppressed because it is too large Load Diff

0
comfyui_manager/glob/utils/__init__.py Normal file
View File

142
comfyui_manager/glob/utils/environment_utils.py Normal file
View File

@@ -0,0 +1,142 @@
import os
import git
import logging
import traceback
from comfyui_manager.common import context
import folder_paths
from comfy.cli_args import args
import latent_preview
from comfyui_manager.glob import manager_core as core
from comfyui_manager.common import cm_global
comfy_ui_hash = "-"
comfyui_tag = None
def print_comfyui_version():
global comfy_ui_hash
global comfyui_tag
is_detached = False
try:
repo = git.Repo(os.path.dirname(folder_paths.__file__))
core.comfy_ui_revision = len(list(repo.iter_commits("HEAD")))
comfy_ui_hash = repo.head.commit.hexsha
cm_global.variables["comfyui.revision"] = core.comfy_ui_revision
core.comfy_ui_commit_datetime = repo.head.commit.committed_datetime
cm_global.variables["comfyui.commit_datetime"] = core.comfy_ui_commit_datetime
is_detached = repo.head.is_detached
current_branch = repo.active_branch.name
comfyui_tag = context.get_comfyui_tag()
try:
if (
not os.environ.get("__COMFYUI_DESKTOP_VERSION__")
and core.comfy_ui_commit_datetime.date()
< core.comfy_ui_required_commit_datetime.date()
):
logging.warning(
f"\n\n## [WARN] ComfyUI-Manager: Your ComfyUI version ({core.comfy_ui_revision})[{core.comfy_ui_commit_datetime.date()}] is too old. Please update to the latest version. ##\n\n"
)
except Exception:
pass
# process on_revision_detected -->
if "cm.on_revision_detected_handler" in cm_global.variables:
for k, f in cm_global.variables["cm.on_revision_detected_handler"]:
try:
f(core.comfy_ui_revision)
except Exception:
logging.error(f"[ERROR] '{k}' on_revision_detected_handler")
traceback.print_exc()
del cm_global.variables["cm.on_revision_detected_handler"]
else:
logging.warning(
"[ComfyUI-Manager] Some features are restricted due to your ComfyUI being outdated."
)
# <--
if current_branch == "master":
if comfyui_tag:
logging.info(
f"### ComfyUI Version: {comfyui_tag} | Released on '{core.comfy_ui_commit_datetime.date()}'"
)
else:
logging.info(
f"### ComfyUI Revision: {core.comfy_ui_revision} [{comfy_ui_hash[:8]}] | Released on '{core.comfy_ui_commit_datetime.date()}'"
)
else:
if comfyui_tag:
logging.info(
f"### ComfyUI Version: {comfyui_tag} on '{current_branch}' | Released on '{core.comfy_ui_commit_datetime.date()}'"
)
else:
logging.info(
f"### ComfyUI Revision: {core.comfy_ui_revision} on '{current_branch}' [{comfy_ui_hash[:8]}] | Released on '{core.comfy_ui_commit_datetime.date()}'"
)
except Exception:
if is_detached:
logging.info(
f"### ComfyUI Revision: {core.comfy_ui_revision} [{comfy_ui_hash[:8]}] *DETACHED | Released on '{core.comfy_ui_commit_datetime.date()}'"
)
else:
logging.info(
"### ComfyUI Revision: UNKNOWN (The currently installed ComfyUI is not a Git repository)"
)
def set_preview_method(method):
if method == "auto":
args.preview_method = latent_preview.LatentPreviewMethod.Auto
elif method == "latent2rgb":
args.preview_method = latent_preview.LatentPreviewMethod.Latent2RGB
elif method == "taesd":
args.preview_method = latent_preview.LatentPreviewMethod.TAESD
else:
args.preview_method = latent_preview.LatentPreviewMethod.NoPreviews
core.get_config()["preview_method"] = method
def set_update_policy(mode):
core.get_config()["update_policy"] = mode
def set_db_mode(mode):
core.get_config()["db_mode"] = mode
def setup_environment():
git_exe = core.get_config()["git_exe"]
if git_exe != "":
git.Git().update_environment(GIT_PYTHON_GIT_EXECUTABLE=git_exe)
def initialize_environment():
context.comfy_path = os.path.dirname(folder_paths.__file__)
core.js_path = os.path.join(context.comfy_path, "web", "extensions")
# Legacy database paths - kept for potential future use
# local_db_model = os.path.join(manager_util.comfyui_manager_path, "model-list.json")
# local_db_alter = os.path.join(manager_util.comfyui_manager_path, "alter-list.json")
# local_db_custom_node_list = os.path.join(
# manager_util.comfyui_manager_path, "custom-node-list.json"
# )
# local_db_extension_node_mappings = os.path.join(
# manager_util.comfyui_manager_path, "extension-node-map.json"
# )
set_preview_method(core.get_config()["preview_method"])
print_comfyui_version()
setup_environment()
core.check_invalid_nodes()

60
comfyui_manager/glob/utils/formatting_utils.py Normal file
View File

@@ -0,0 +1,60 @@
import locale
import sys
import re
def handle_stream(stream, prefix):
stream.reconfigure(encoding=locale.getpreferredencoding(), errors="replace")
for msg in stream:
if (
prefix == "[!]"
and ("it/s]" in msg or "s/it]" in msg)
and ("%|" in msg or "it [" in msg)
):
if msg.startswith("100%"):
print("\r" + msg, end="", file=sys.stderr),
else:
print("\r" + msg[:-1], end="", file=sys.stderr),
else:
if prefix == "[!]":
print(prefix, msg, end="", file=sys.stderr)
else:
print(prefix, msg, end="")
def convert_markdown_to_html(input_text):
pattern_a = re.compile(r"\[a/([^]]+)]\(([^)]+)\)")
pattern_w = re.compile(r"\[w/([^]]+)]")
pattern_i = re.compile(r"\[i/([^]]+)]")
pattern_bold = re.compile(r"\*\*([^*]+)\*\*")
pattern_white = re.compile(r"%%([^*]+)%%")
def replace_a(match):
return f"<a href='{match.group(2)}' target='blank'>{match.group(1)}</a>"
def replace_w(match):
return f"<p class='cm-warn-note'>{match.group(1)}</p>"
def replace_i(match):
return f"<p class='cm-info-note'>{match.group(1)}</p>"
def replace_bold(match):
return f"<B>{match.group(1)}</B>"
def replace_white(match):
return f"<font color='white'>{match.group(1)}</font>"
input_text = (
input_text.replace("\\[", "&#91;")
.replace("\\]", "&#93;")
.replace("<", "&lt;")
.replace(">", "&gt;")
)
result_text = re.sub(pattern_a, replace_a, input_text)
result_text = re.sub(pattern_w, replace_w, result_text)
result_text = re.sub(pattern_i, replace_i, result_text)
result_text = re.sub(pattern_bold, replace_bold, result_text)
result_text = re.sub(pattern_white, replace_white, result_text)
return result_text.replace("\n", "<BR>")

161
comfyui_manager/glob/utils/model_utils.py Normal file
View File

@@ -0,0 +1,161 @@
import os
import logging
import concurrent.futures
import folder_paths
from comfyui_manager.glob import manager_core as core
from comfyui_manager.glob.constants import model_dir_name_map, MODEL_DIR_NAMES
def get_model_dir(data, show_log=False):
if "download_model_base" in folder_paths.folder_names_and_paths:
models_base = folder_paths.folder_names_and_paths["download_model_base"][0][0]
else:
models_base = folder_paths.models_dir
# NOTE: Validate to prevent path traversal.
if any(char in data["filename"] for char in {"/", "\\", ":"}):
return None
def resolve_custom_node(save_path):
save_path = save_path[13:] # remove 'custom_nodes/'
# NOTE: Validate to prevent path traversal.
if save_path.startswith(os.path.sep) or ":" in save_path:
return None
repo_name = save_path.replace("\\", "/").split("/")[
0
] # get custom node repo name
# NOTE: The creation of files within the custom node path should be removed in the future.
repo_path = core.lookup_installed_custom_nodes_legacy(repo_name)
if repo_path is not None and repo_path[0]:
# Returns the retargeted path based on the actually installed repository
return os.path.join(os.path.dirname(repo_path[1]), save_path)
else:
return None
if data["save_path"] != "default":
if ".." in data["save_path"] or data["save_path"].startswith("/"):
if show_log:
logging.info(
f"[WARN] '{data['save_path']}' is not allowed path. So it will be saved into 'models/etc'."
)
base_model = os.path.join(models_base, "etc")
else:
if data["save_path"].startswith("custom_nodes"):
base_model = resolve_custom_node(data["save_path"])
if base_model is None:
if show_log:
logging.info(
f"[ComfyUI-Manager] The target custom node for model download is not installed: {data['save_path']}"
)
return None
else:
base_model = os.path.join(models_base, data["save_path"])
else:
model_dir_name = model_dir_name_map.get(data["type"].lower())
if model_dir_name is not None:
base_model = folder_paths.folder_names_and_paths[model_dir_name][0][0]
else:
base_model = os.path.join(models_base, "etc")
return base_model
def get_model_path(data, show_log=False):
base_model = get_model_dir(data, show_log)
if base_model is None:
return None
else:
if data["filename"] == "<huggingface>":
return os.path.join(base_model, os.path.basename(data["url"]))
else:
return os.path.join(base_model, data["filename"])
def check_model_installed(json_obj):
def is_exists(model_dir_name, filename, url):
if filename == "<huggingface>":
filename = os.path.basename(url)
dirs = folder_paths.get_folder_paths(model_dir_name)
for x in dirs:
if os.path.exists(os.path.join(x, filename)):
return True
return False
total_models_files = set()
for x in MODEL_DIR_NAMES:
for y in folder_paths.get_filename_list(x):
total_models_files.add(y)
def process_model_phase(item):
if (
"diffusion" not in item["filename"]
and "pytorch" not in item["filename"]
and "model" not in item["filename"]
):
# non-general name case
if item["filename"] in total_models_files:
item["installed"] = "True"
return
if item["save_path"] == "default":
model_dir_name = model_dir_name_map.get(item["type"].lower())
if model_dir_name is not None:
item["installed"] = str(
is_exists(model_dir_name, item["filename"], item["url"])
)
else:
item["installed"] = "False"
else:
model_dir_name = item["save_path"].split("/")[0]
if model_dir_name in folder_paths.folder_names_and_paths:
if is_exists(model_dir_name, item["filename"], item["url"]):
item["installed"] = "True"
if "installed" not in item:
if item["filename"] == "<huggingface>":
filename = os.path.basename(item["url"])
else:
filename = item["filename"]
fullpath = os.path.join(
folder_paths.models_dir, item["save_path"], filename
)
item["installed"] = "True" if os.path.exists(fullpath) else "False"
with concurrent.futures.ThreadPoolExecutor(8) as executor:
for item in json_obj["models"]:
executor.submit(process_model_phase, item)
async def check_whitelist_for_model(item):
from comfyui_manager.data_models import ManagerDatabaseSource
json_obj = await core.get_data_by_mode(ManagerDatabaseSource.cache.value, "model-list.json")
for x in json_obj.get("models", []):
if (
x["save_path"] == item["save_path"]
and x["base"] == item["base"]
and x["filename"] == item["filename"]
):
return True
json_obj = await core.get_data_by_mode(ManagerDatabaseSource.local.value, "model-list.json")
for x in json_obj.get("models", []):
if (
x["save_path"] == item["save_path"]
and x["base"] == item["base"]
and x["filename"] == item["filename"]
):
return True
return False

65
comfyui_manager/glob/utils/node_pack_utils.py Normal file
View File

@@ -0,0 +1,65 @@
import concurrent.futures
from comfyui_manager.glob import manager_core as core
def check_state_of_git_node_pack(
node_packs, do_fetch=False, do_update_check=True, do_update=False
):
if do_fetch:
print("Start fetching...", end="")
elif do_update:
print("Start updating...", end="")
elif do_update_check:
print("Start update check...", end="")
def process_custom_node(item):
core.check_state_of_git_node_pack_single(
item, do_fetch, do_update_check, do_update
)
with concurrent.futures.ThreadPoolExecutor(4) as executor:
for k, v in node_packs.items():
if v.get("active_version") in ["unknown", "nightly"]:
executor.submit(process_custom_node, v)
if do_fetch:
print("\x1b[2K\rFetching done.")
elif do_update:
update_exists = any(
item.get("updatable", False) for item in node_packs.values()
)
if update_exists:
print("\x1b[2K\rUpdate done.")
else:
print("\x1b[2K\rAll extensions are already up-to-date.")
elif do_update_check:
print("\x1b[2K\rUpdate check done.")
def nickname_filter(json_obj):
preemptions_map = {}
for k, x in json_obj.items():
if "preemptions" in x[1]:
for y in x[1]["preemptions"]:
preemptions_map[y] = k
elif k.endswith("/ComfyUI"):
for y in x[0]:
preemptions_map[y] = k
updates = {}
for k, x in json_obj.items():
removes = set()
for y in x[0]:
k2 = preemptions_map.get(y)
if k2 is not None and k != k2:
removes.add(y)
if len(removes) > 0:
updates[k] = [y for y in x[0] if y not in removes]
for k, v in updates.items():
json_obj[k][0] = v
return json_obj

54
comfyui_manager/glob/utils/security_utils.py Normal file
View File

@@ -0,0 +1,54 @@
from comfyui_manager.glob import manager_core as core
from comfy.cli_args import args
from comfyui_manager.data_models import SecurityLevel, RiskLevel, ManagerDatabaseSource
def is_loopback(address):
import ipaddress
try:
return ipaddress.ip_address(address).is_loopback
except ValueError:
return False
def is_allowed_security_level(level):
is_local_mode = is_loopback(args.listen)
if level == RiskLevel.block.value:
return False
elif level == RiskLevel.high.value:
if is_local_mode:
return core.get_config()["security_level"] in [SecurityLevel.weak.value, SecurityLevel.normal_.value]
else:
return core.get_config()["security_level"] == SecurityLevel.weak.value
elif level == RiskLevel.middle.value:
return core.get_config()["security_level"] in [SecurityLevel.weak.value, SecurityLevel.normal.value, SecurityLevel.normal_.value]
else:
return True
async def get_risky_level(files, pip_packages):
json_data1 = await core.get_data_by_mode(ManagerDatabaseSource.local.value, "custom-node-list.json")
json_data2 = await core.get_data_by_mode(
ManagerDatabaseSource.cache.value,
"custom-node-list.json",
channel_url="https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main",
)
all_urls = set()
for x in json_data1["custom_nodes"] + json_data2["custom_nodes"]:
all_urls.update(x.get("files", []))
for x in files:
if x not in all_urls:
return RiskLevel.high.value
all_pip_packages = set()
for x in json_data1["custom_nodes"] + json_data2["custom_nodes"]:
all_pip_packages.update(x.get("pip", []))
for p in pip_packages:
if p not in all_pip_packages:
return RiskLevel.block.value
return RiskLevel.middle.value