prepare release 2.1.4

Optimize models request API

__init__.py

@@ -23,7 +23,6 @@ if len(uninstalled_package) > 0:
 
 # Init config settings
 config.extension_uri = extension_uri
-utils.resolve_model_base_paths()
 
 version = utils.get_current_version()
 utils.download_web_distribution(version)
@@ -90,12 +89,13 @@ async def delete_model_download_task(request):
         return web.json_response({"success": False, "error": error_msg})
 
 
+# @deprecated
 @routes.get("/model-manager/base-folders")
 async def get_model_paths(request):
     """
     Returns the base folders for models.
     """
-    model_base_paths = config.model_base_paths
+    model_base_paths = utils.resolve_model_base_paths()
     return web.json_response({"success": True, "data": model_base_paths})
 
 
@@ -125,12 +125,12 @@ async def create_model(request):
 
 
 @routes.get("/model-manager/models")
-async def read_models(request):
+async def list_model_types(request):
     """
     Scan all models and read their information.
     """
     try:
-        result = services.scan_models()
+        result = utils.resolve_model_base_paths()
         return web.json_response({"success": True, "data": result})
     except Exception as e:
         error_msg = f"Read models failed: {str(e)}"
@@ -138,6 +138,18 @@ async def read_models(request):
         return web.json_response({"success": False, "error": error_msg})
 
 
+@routes.get("/model-manager/models/{folder}")
+async def read_models(request):
+    try:
+        folder = request.match_info.get("folder", None)
+        results = services.scan_models(folder, request)
+        return web.json_response({"success": True, "data": results})
+    except Exception as e:
+        error_msg = f"Read models failed: {str(e)}"
+        utils.print_error(error_msg)
+        return web.json_response({"success": False, "error": error_msg})
+
+
 @routes.get("/model-manager/model/{type}/{index}/{filename:.*}")
 async def read_model_info(request):
     """
@@ -232,7 +244,7 @@ async def download_model_info(request):
     post = await utils.get_request_body(request)
     try:
         scan_mode = post.get("scanMode", "diff")
-        await services.download_model_info(scan_mode)
+        await services.download_model_info(scan_mode, request)
         return web.json_response({"success": True})
     except Exception as e:
         error_msg = f"Download model info failed: {str(e)}"
@@ -288,10 +300,10 @@ async def migrate_legacy_information(request):
     Migrate legacy information.
     """
     try:
-        await services.migrate_legacy_information()
+        await services.migrate_legacy_information(request)
         return web.json_response({"success": True})
     except Exception as e:
-        error_msg = f"Download model info failed: {str(e)}"
+        error_msg = f"Migrate model info failed: {str(e)}"
         utils.print_error(error_msg)
         return web.json_response({"success": False, "error": error_msg})
 

py/config.py

@@ -1,7 +1,6 @@
 extension_tag = "ComfyUI Model Manager"
 
 extension_uri: str = None
-model_base_paths: dict[str, list[str]] = {}
 
 
 setting_key = {
@@ -12,6 +11,9 @@ setting_key = {
     "download": {
         "max_task_count": "ModelManager.Download.MaxTaskCount",
     },
+    "scan": {
+        "include_hidden_files": "ModelManager.Scan.IncludeHiddenFiles"
+    },
 }
 
 user_agent = "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"

py/services.py

@@ -8,49 +8,46 @@ from . import download
 from . import searcher
 
 
-def scan_models():
+def scan_models(folder: str, request):
     result = []
-    model_base_paths = config.model_base_paths
-    for model_type in model_base_paths:
 
-        folders, extensions = folder_paths.folder_names_and_paths[model_type]
-        for path_index, base_path in enumerate(folders):
-            files = utils.recursive_search_files(base_path)
+    folders, extensions = folder_paths.folder_names_and_paths[folder]
+    for path_index, base_path in enumerate(folders):
+        files = utils.recursive_search_files(base_path, request)
 
-            models = folder_paths.filter_files_extensions(files, extensions)
+        models = folder_paths.filter_files_extensions(files, folder_paths.supported_pt_extensions)
 
-            for fullname in models:
-                fullname = utils.normalize_path(fullname)
-                basename = os.path.splitext(fullname)[0]
-                extension = os.path.splitext(fullname)[1]
+        for fullname in models:
+            fullname = utils.normalize_path(fullname)
+            basename = os.path.splitext(fullname)[0]
+            extension = os.path.splitext(fullname)[1]
 
-                abs_path = utils.join_path(base_path, fullname)
-                file_stats = os.stat(abs_path)
+            abs_path = utils.join_path(base_path, fullname)
+            file_stats = os.stat(abs_path)
 
-                # Resolve preview
-                image_name = utils.get_model_preview_name(abs_path)
-                abs_image_path = utils.join_path(base_path, image_name)
-                if os.path.isfile(abs_image_path):
-                    image_state = os.stat(abs_image_path)
-                    image_timestamp = round(image_state.st_mtime_ns / 1000000)
-                    image_name = f"{image_name}?ts={image_timestamp}"
-                model_preview = (
-                    f"/model-manager/preview/{model_type}/{path_index}/{image_name}"
-                )
+            # Resolve preview
+            image_name = utils.get_model_preview_name(abs_path)
+            image_name = utils.join_path(os.path.dirname(fullname), image_name)
+            abs_image_path = utils.join_path(base_path, image_name)
+            if os.path.isfile(abs_image_path):
+                image_state = os.stat(abs_image_path)
+                image_timestamp = round(image_state.st_mtime_ns / 1000000)
+                image_name = f"{image_name}?ts={image_timestamp}"
+            model_preview = f"/model-manager/preview/{folder}/{path_index}/{image_name}"
 
-                model_info = {
-                    "fullname": fullname,
-                    "basename": basename,
-                    "extension": extension,
-                    "type": model_type,
-                    "pathIndex": path_index,
-                    "sizeBytes": file_stats.st_size,
-                    "preview": model_preview,
-                    "createdAt": round(file_stats.st_ctime_ns / 1000000),
-                    "updatedAt": round(file_stats.st_mtime_ns / 1000000),
-                }
+            model_info = {
+                "fullname": fullname,
+                "basename": basename,
+                "extension": extension,
+                "type": folder,
+                "pathIndex": path_index,
+                "sizeBytes": file_stats.st_size,
+                "preview": model_preview,
+                "createdAt": round(file_stats.st_ctime_ns / 1000000),
+                "updatedAt": round(file_stats.st_mtime_ns / 1000000),
+            }
 
-                result.append(model_info)
+            result.append(model_info)
 
     return result
 
@@ -138,16 +135,16 @@ def fetch_model_info(model_page: str):
     return result
 
 
-async def download_model_info(scan_mode: str):
+async def download_model_info(scan_mode: str, request):
     utils.print_info(f"Download model info for {scan_mode}")
-    model_base_paths = config.model_base_paths
+    model_base_paths = utils.resolve_model_base_paths()
     for model_type in model_base_paths:
 
         folders, extensions = folder_paths.folder_names_and_paths[model_type]
         for path_index, base_path in enumerate(folders):
-            files = utils.recursive_search_files(base_path)
+            files = utils.recursive_search_files(base_path, request)
 
-            models = folder_paths.filter_files_extensions(files, extensions)
+            models = folder_paths.filter_files_extensions(files, folder_paths.supported_pt_extensions)
 
             for fullname in models:
                 fullname = utils.normalize_path(fullname)
@@ -161,16 +158,8 @@ async def download_model_info(scan_mode: str):
                 has_preview = os.path.isfile(abs_image_path)
 
                 description_name = utils.get_model_description_name(abs_model_path)
-                abs_description_path = (
-                    utils.join_path(base_path, description_name)
-                    if description_name
-                    else None
-                )
-                has_description = (
-                    os.path.isfile(abs_description_path)
-                    if abs_description_path
-                    else False
-                )
+                abs_description_path = utils.join_path(base_path, description_name) if description_name else None
+                has_description = os.path.isfile(abs_description_path) if abs_description_path else False
 
                 try:
 
@@ -185,46 +174,38 @@ async def download_model_info(scan_mode: str):
                     utils.print_debug(f"Calculate sha256 for {abs_model_path}")
                     hash_value = utils.calculate_sha256(abs_model_path)
                     utils.print_info(f"Searching model info by hash {hash_value}")
-                    model_info = searcher.CivitaiModelSearcher().search_by_hash(
-                        hash_value
-                    )
+                    model_info = searcher.CivitaiModelSearcher().search_by_hash(hash_value)
 
                     preview_url_list = model_info.get("preview", [])
-                    preview_image_url = (
-                        preview_url_list[0] if preview_url_list else None
-                    )
+                    preview_image_url = preview_url_list[0] if preview_url_list else None
                     if preview_image_url:
                         utils.print_debug(f"Save preview image to {abs_image_path}")
-                        utils.save_model_preview_image(
-                            abs_model_path, preview_image_url
-                        )
+                        utils.save_model_preview_image(abs_model_path, preview_image_url)
 
                     description = model_info.get("description", None)
                     if description:
                         utils.save_model_description(abs_model_path, description)
                 except Exception as e:
-                    utils.print_error(
-                        f"Failed to download model info for {abs_model_path}: {e}"
-                    )
+                    utils.print_error(f"Failed to download model info for {abs_model_path}: {e}")
 
     utils.print_debug("Completed scan model information.")
 
 
-async def migrate_legacy_information():
+async def migrate_legacy_information(request):
     import json
     import yaml
     from PIL import Image
 
     utils.print_info(f"Migrating legacy information...")
 
-    model_base_paths = config.model_base_paths
+    model_base_paths = utils.resolve_model_base_paths()
     for model_type in model_base_paths:
 
         folders, extensions = folder_paths.folder_names_and_paths[model_type]
         for path_index, base_path in enumerate(folders):
-            files = utils.recursive_search_files(base_path)
+            files = utils.recursive_search_files(base_path, request)
 
-            models = folder_paths.filter_files_extensions(files, extensions)
+            models = folder_paths.filter_files_extensions(files, folder_paths.supported_pt_extensions)
 
             for fullname in models:
                 fullname = utils.normalize_path(fullname)
@@ -290,5 +271,4 @@ async def migrate_legacy_information():
                     with open(description_path, "w", encoding="utf-8", newline="") as f:
                         f.write("\n".join(description_parts))
 
-
     utils.print_debug("Completed migrate model information.")

py/utils.py

@@ -103,9 +103,7 @@ def download_web_distribution(version: str):
 
         print_info("Extracting web distribution...")
         with tarfile.open(temp_file, "r:gz") as tar:
-            members = [
-                member for member in tar.getmembers() if member.name.startswith("web/")
-            ]
+            members = [member for member in tar.getmembers() if member.name.startswith("web/")]
             tar.extractall(path=config.extension_uri, members=members)
 
         os.remove(temp_file)
@@ -120,21 +118,21 @@ def download_web_distribution(version: str):
 
 def resolve_model_base_paths():
     folders = list(folder_paths.folder_names_and_paths.keys())
-    config.model_base_paths = {}
+    model_base_paths = {}
+    folder_black_list = ["configs", "custom_nodes"]
     for folder in folders:
-        if folder == "configs":
-            continue
-        if folder == "custom_nodes":
+        if folder in folder_black_list:
             continue
         folders = folder_paths.get_folder_paths(folder)
-        config.model_base_paths[folder] = [normalize_path(f) for f in folders]
+        model_base_paths[folder] = [normalize_path(f) for f in folders]
+    return model_base_paths
 
 
 def get_full_path(model_type: str, path_index: int, filename: str):
     """
     Get the absolute path in the model type through string concatenation.
     """
-    folders = config.model_base_paths.get(model_type, [])
+    folders = resolve_model_base_paths().get(model_type, [])
     if not path_index < len(folders):
         raise RuntimeError(f"PathIndex {path_index} is not in {model_type}")
     base_path = folders[path_index]
@@ -146,7 +144,7 @@ def get_valid_full_path(model_type: str, path_index: int, filename: str):
     """
     Like get_full_path but it will check whether the file is valid.
     """
-    folders = config.model_base_paths.get(model_type, [])
+    folders = resolve_model_base_paths().get(model_type, [])
     if not path_index < len(folders):
         raise RuntimeError(f"PathIndex {path_index} is not in {model_type}")
     base_path = folders[path_index]
@@ -154,9 +152,7 @@ def get_valid_full_path(model_type: str, path_index: int, filename: str):
     if os.path.isfile(full_path):
         return full_path
     elif os.path.islink(full_path):
-        raise RuntimeError(
-            f"WARNING path {full_path} exists but doesn't link anywhere, skipping."
-        )
+        raise RuntimeError(f"WARNING path {full_path} exists but doesn't link anywhere, skipping.")
 
 
 def get_download_path():
@@ -166,11 +162,29 @@ def get_download_path():
     return download_path
 
 
-def recursive_search_files(directory: str):
-    files, folder_all = folder_paths.recursive_search(
-        directory, excluded_dir_names=[".git"]
-    )
-    return [normalize_path(f) for f in files]
+def recursive_search_files(directory: str, request):
+    if not os.path.isdir(directory):
+        return []
+
+    excluded_dir_names = [".git"]
+    result = []
+    include_hidden_files = get_setting_value(request, "scan.include_hidden_files", False)
+
+    for dirpath, subdirs, filenames in os.walk(directory, followlinks=True, topdown=True):
+        subdirs[:] = [d for d in subdirs if d not in excluded_dir_names]
+        if not include_hidden_files:
+            subdirs[:] = [d for d in subdirs if not d.startswith(".")]
+            filenames[:] = [f for f in filenames if not f.startswith(".")]
+
+        for file_name in filenames:
+            try:
+                relative_path = os.path.relpath(os.path.join(dirpath, file_name), directory)
+                result.append(relative_path)
+            except:
+                logging.warning(f"Warning: Unable to access {file_name}. Skipping this file.")
+                continue
+
+    return [normalize_path(f) for f in result]
 
 
 def search_files(directory: str):

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-model-manager"
 description = "Manage models: browsing, download and delete."
-version = "2.1.1"
+version = "2.1.4"
 license = "LICENSE"
 dependencies = ["markdownify"]
 
@@ -13,3 +13,6 @@ Repository = "https://github.com/hayden-fr/ComfyUI-Model-Manager"
 PublisherId = "hayden"
 DisplayName = "ComfyUI-Model-Manager"
 Icon = ""
+
+[tool.black]
+line-length = 160

src/App.vue

@@ -15,16 +15,18 @@ import { useStoreProvider } from 'hooks/store'
 import { useToast } from 'hooks/toast'
 import GlobalConfirm from 'primevue/confirmdialog'
 import { $el, app, ComfyButton } from 'scripts/comfyAPI'
-import { onMounted } from 'vue'
+import { onMounted, ref } from 'vue'
 import { useI18n } from 'vue-i18n'
 
 const { t } = useI18n()
 const { dialog, models, config, download } = useStoreProvider()
 const { toast } = useToast()
 
+const firstOpenManager = ref(true)
+
 onMounted(() => {
   const refreshModelsAndConfig = async () => {
-    await Promise.all([models.refresh(), config.refresh()])
+    await Promise.all([models.refresh(true)])
     toast.add({
       severity: 'success',
       summary: 'Refreshed Models',
@@ -50,6 +52,11 @@ onMounted(() => {
   const openManagerDialog = () => {
     const { cardWidth, gutter, aspect } = config
 
+    if (firstOpenManager.value) {
+      models.refresh(true)
+      firstOpenManager.value = false
+    }
+
     dialog.open({
       key: 'model-manager',
       title: t('modelManager'),

src/components/DialogManager.vue

@@ -88,9 +88,9 @@ import { genModelKey } from 'utils/model'
 import { computed, ref, watch } from 'vue'
 import { useI18n } from 'vue-i18n'
 
-const { isMobile, cardWidth, gutter, aspect, modelFolders } = useConfig()
+const { isMobile, cardWidth, gutter, aspect } = useConfig()
 
-const { data } = useModels()
+const { data, folders } = useModels()
 const { t } = useI18n()
 
 const responseScroll = ref()
@@ -99,7 +99,7 @@ const searchContent = ref<string>()
 
 const currentType = ref('all')
 const typeOptions = computed(() => {
-  return ['all', ...Object.keys(modelFolders.value)].map((type) => {
+  return ['all', ...Object.keys(folders.value)].map((type) => {
     return {
       label: type,
       value: type,
@@ -143,7 +143,9 @@ const colSpan = ref(1)
 const colSpanWidth = ref(cardWidth)
 
 const list = computed(() => {
-  const filterList = data.value.filter((model) => {
+  const mergedList = Object.values(data.value).flat()
+
+  const filterList = mergedList.filter((model) => {
     const showAllModel = currentType.value === 'all'
 
     const matchType = showAllModel || model.type === currentType.value

src/components/ModelBaseInfo.vue

@@ -49,15 +49,13 @@
 <script setup lang="ts">
 import ResponseInput from 'components/ResponseInput.vue'
 import ResponseSelect from 'components/ResponseSelect.vue'
-import { useConfig } from 'hooks/config'
 import { useModelBaseInfo } from 'hooks/model'
 import { computed } from 'vue'
 
 const editable = defineModel<boolean>('editable')
 
-const { modelFolders } = useConfig()
-
-const { baseInfo, pathIndex, basename, extension, type } = useModelBaseInfo()
+const { baseInfo, pathIndex, basename, extension, type, modelFolders } =
+  useModelBaseInfo()
 
 const typeOptions = computed(() => {
   return Object.keys(modelFolders.value).map((curr) => {

src/hooks/config.ts

@@ -1,4 +1,4 @@
-import { request, useRequest } from 'hooks/request'
+import { request } from 'hooks/request'
 import { defineStore } from 'hooks/store'
 import { $el, app, ComfyDialog } from 'scripts/comfyAPI'
 import { onMounted, onUnmounted, ref } from 'vue'
@@ -8,10 +8,6 @@ export const useConfig = defineStore('config', (store) => {
   const mobileDeviceBreakPoint = 759
   const isMobile = ref(window.innerWidth < mobileDeviceBreakPoint)
 
-  type ModelFolder = Record<string, string[]>
-  const { data: modelFolders, refresh: refreshModelFolders } =
-    useRequest<ModelFolder>('/base-folders')
-
   const checkDeviceType = () => {
     isMobile.value = window.innerWidth < mobileDeviceBreakPoint
   }
@@ -24,17 +20,11 @@ export const useConfig = defineStore('config', (store) => {
     window.removeEventListener('resize', checkDeviceType)
   })
 
-  const refresh = async () => {
-    return Promise.all([refreshModelFolders()])
-  }
-
   const config = {
     isMobile,
     gutter: 16,
     cardWidth: 240,
     aspect: 7 / 9,
-    modelFolders,
-    refresh,
   }
 
   useAddConfigSettings(store)
@@ -239,5 +229,12 @@ function useAddConfigSettings(store: import('hooks/store').StoreProvider) {
         })
       },
     })
+
+    app.ui?.settings.addSetting({
+      id: 'ModelManager.Scan.IncludeHiddenFiles',
+      name: 'Include hidden files(start with .)',
+      defaultValue: false,
+      type: 'boolean',
+    })
   })
 }

src/hooks/model.ts

@@ -1,7 +1,6 @@
-import { useConfig } from 'hooks/config'
 import { useLoading } from 'hooks/loading'
 import { useMarkdown } from 'hooks/markdown'
-import { request, useRequest } from 'hooks/request'
+import { request } from 'hooks/request'
 import { defineStore } from 'hooks/store'
 import { useToast } from 'hooks/toast'
 import { cloneDeep } from 'lodash'
@@ -22,12 +21,45 @@ import {
 } from 'vue'
 import { useI18n } from 'vue-i18n'
 
+type ModelFolder = Record<string, string[]>
+
+const modelFolderProvideKey = Symbol('modelFolder')
+
 export const useModels = defineStore('models', (store) => {
-  const { data, refresh } = useRequest<Model[]>('/models', { defaultValue: [] })
   const { toast, confirm } = useToast()
   const { t } = useI18n()
   const loading = useLoading()
 
+  const folders = ref<ModelFolder>({})
+  const refreshFolders = async () => {
+    return request('/models').then((resData) => {
+      folders.value = resData
+    })
+  }
+
+  provide(modelFolderProvideKey, folders)
+
+  const models = ref<Record<string, Model[]>>({})
+
+  const refreshModels = async (folder: string) => {
+    loading.show()
+    return request(`/models/${folder}`)
+      .then((resData) => {
+        models.value[folder] = resData
+        return resData
+      })
+      .finally(() => {
+        loading.hide()
+      })
+  }
+
+  const refreshAllModels = async (force = false) => {
+    const forceRefresh = force ? refreshFolders() : Promise.resolve()
+    return forceRefresh.then(() =>
+      Promise.allSettled(Object.keys(folders.value).map(refreshModels)),
+    )
+  }
+
   const updateModel = async (model: BaseModel, data: BaseModel) => {
     const updateData = new Map()
     let oldKey: string | null = null
@@ -80,7 +112,7 @@ export const useModels = defineStore('models', (store) => {
       store.dialog.close({ key: oldKey })
     }
 
-    refresh()
+    refreshModels(data.type)
   }
 
   const deleteModel = async (model: BaseModel) => {
@@ -112,7 +144,7 @@ export const useModels = defineStore('models', (store) => {
                 life: 2000,
               })
               store.dialog.close({ key: dialogKey })
-              return refresh()
+              return refreshModels(model.type)
             })
             .then(() => {
               resolve(void 0)
@@ -136,7 +168,13 @@ export const useModels = defineStore('models', (store) => {
     })
   }
 
-  return { data, refresh, remove: deleteModel, update: updateModel }
+  return {
+    folders: folders,
+    data: models,
+    refresh: refreshAllModels,
+    remove: deleteModel,
+    update: updateModel,
+  }
 })
 
 declare module 'hooks/store' {
@@ -204,7 +242,10 @@ const baseInfoKey = Symbol('baseInfo') as InjectionKey<
 export const useModelBaseInfoEditor = (formInstance: ModelFormInstance) => {
   const { formData: model, modelData } = formInstance
 
-  const { modelFolders } = useConfig()
+  const provideModelFolders = inject<any>(modelFolderProvideKey)
+  const modelFolders = computed<ModelFolder>(() => {
+    return provideModelFolders?.value ?? {}
+  })
 
   const type = computed({
     get: () => {
@@ -304,6 +345,7 @@ export const useModelBaseInfoEditor = (formInstance: ModelFormInstance) => {
     basename,
     extension,
     pathIndex,
+    modelFolders,
   }
 
   provide(baseInfoKey, result)