* FIXED: Resolved an issue where cache updates were not working properly.

* IMPROVED: Instead of updating the entire CNR cache at once, the process now divides it into 30-page queries.
* IMPROVED: Clicking on the titles of nodes that exist only in CNR now opens the GitHub repository link instead of the CNR link, where possible.
* ADDED: Added information about `extra_model_paths.yaml` to the README.md file.

Comfy-Org#1457

Author: ltdrdata

README.md

@@ -146,7 +146,15 @@ In `ComfyUI-Manager` V3.0 and later, configuration files and dynamically generat
 * Saved snapshot files: `<USER_DIRECTORY>/default/ComfyUI-Manager/snapshots`
 * Startup script files: `<USER_DIRECTORY>/default/ComfyUI-Manager/startup-scripts`
 * Component files: `<USER_DIRECTORY>/default/ComfyUI-Manager/components`
-  
+
+
+## `extra_model_paths.yaml` Configuration
+The following settings are applied based on the section marked as `is_default`.
+
+* `custom_nodes`: Path for installing custom nodes
+    * Importing does not need to adhere to the path set as `is_default`, but this is the path where custom nodes are installed by the `ComfyUI Nodes Manager`.
+* `download_model_base`: Path for downloading models
+
 
 ## Snapshot-Manager
 * When you press `Save snapshot` or use `Update All` on `Manager Menu`, the current installation status snapshot is saved.

extras.json

@@ -9,7 +9,7 @@
         "comfyui-layerdiffuse",
         "comfyui-liveportraitkj",
         "aigodlike-comfyui-translation",
-        "comfyui-reactor-node",
+        "comfyui-reactor",
         "comfyui_instantid",
         "sd-dynamic-thresholding",
         "pr-was-node-suite-comfyui-47064894",

glob/cnr_utils.py

@@ -5,6 +5,7 @@
 import toml
 import os
 import asyncio
+import json
 
 base_url = "https://api.comfy.org"
 
@@ -13,33 +14,55 @@
 
 is_cache_loading = False
 
-async def get_cnr_data(page=1, limit=1000, cache_mode=True, dont_wait=True):
+async def get_cnr_data(cache_mode=True, dont_wait=True):
     global is_cache_loading
 
-    uri = f'{base_url}/nodes?page={page}&limit={limit}'
+    uri = f'{base_url}/nodes'
 
-    def touch(json_obj):
-        for v in json_obj['nodes']:
+    async def fetch_all():
+        remained = True
+        page = 1
+
+        full_nodes = {}
+        while remained:
+            sub_uri = f'{base_url}/nodes?page={page}&limit=30'
+            sub_json_obj = await manager_util.get_data_with_cache(sub_uri, cache_mode=False, silent=True)
+            remained = page < sub_json_obj['totalPages']
+
+            for x in sub_json_obj['nodes']:
+                full_nodes[x['id']] = x
+
+            if page % 5 == 0:
+                print(f"FETCH ComfyRegistry Data: {page}/{sub_json_obj['totalPages']}")
+            page += 1
+
+        print("FETCH ComfyRegistry Data [DONE]")
+
+        for v in full_nodes.values():
             if 'latest_version' not in v:
                 v['latest_version'] = dict(version='nightly')
 
+        return {'nodes': list(full_nodes.values())}
 
     if cache_mode:
-        if dont_wait:
-            json_obj = await manager_util.get_data_with_cache(uri, cache_mode=cache_mode, dont_wait=True)  # fallback
+        is_cache_loading = True
+        cache_state = manager_util.get_cache_state(uri)
 
-            if 'nodes' in json_obj:
-                touch(json_obj)
-                return json_obj['nodes']
-            else:
+        if dont_wait:
+            if cache_state == 'not-cached':
                 return {}
+            else:
+                print("[ComfyUI-Manager] The ComfyRegistry cache update is still in progress, so an outdated cache is being used.")
+                with open(manager_util.get_cache_path(uri), 'r', encoding="UTF-8", errors="ignore") as json_file:
+                    return json.load(json_file)['nodes']
 
-        is_cache_loading = True
+        if cache_state == 'cached':
+            with open(manager_util.get_cache_path(uri), 'r', encoding="UTF-8", errors="ignore") as json_file:
+                return json.load(json_file)['nodes']
 
     try:
-        json_obj = await manager_util.get_data_with_cache(uri, cache_mode=cache_mode)
-        touch(json_obj)
-
+        json_obj = await fetch_all()
+        manager_util.save_to_cache(uri, json_obj)
         return json_obj['nodes']
     except:
         res = {}

glob/manager_core.py

@@ -41,7 +41,7 @@
 from node_package import InstalledNodePackage
 
 
-version_code = [3, 8, 1]
+version_code = [3, 9]
 version_str = f"V{version_code[0]}.{version_code[1]}" + (f'.{version_code[2]}' if len(version_code) > 2 else '')
 
 
@@ -682,11 +682,10 @@ async def reload(self, cache_mode, dont_wait=True):
         self.active_nodes = {}            # node_id -> node_version * fullpath
 
         # reload 'cnr_map' and 'repo_cnr_map'
-        cnrs = await cnr_utils.get_cnr_data(cache_mode=cache_mode, dont_wait=dont_wait)
+        cnrs = await cnr_utils.get_cnr_data(cache_mode=cache_mode=='cache', dont_wait=dont_wait)
 
         for x in cnrs:
             self.cnr_map[x['id']] = x
-
             if 'repository' in x:
                 normalized_url = git_utils.normalize_url(x['repository'])
                 self.repo_cnr_map[normalized_url] = x
@@ -2696,8 +2695,8 @@ def map_to_unified_keys(json_obj):
     return res
 
 
-async def get_unified_total_nodes(channel, mode):
-    await unified_manager.reload(mode)
+async def get_unified_total_nodes(channel, mode, regsitry_cache_mode='cache'):
+    await unified_manager.reload(regsitry_cache_mode)
 
     res = await unified_manager.get_custom_nodes(channel, mode)
 
@@ -2780,6 +2779,7 @@ async def get_unified_total_nodes(channel, mode):
             author = cnr['publisher']['name']
             title = cnr['name']
             reference = f"https://registry.comfy.org/nodes/{cnr['id']}"
+            repository = cnr.get('repository', '')
             install_type = "cnr"
             description = cnr.get('description', '')
 
@@ -2811,7 +2811,7 @@ async def get_unified_total_nodes(channel, mode):
             if ver is None:
                 ver = cnr['latest_version']['version']
 
-            item = dict(author=author, title=title, reference=reference, install_type=install_type,
+            item = dict(author=author, title=title, reference=reference, repository=repository, install_type=install_type,
                         description=description, state=state, updatable=updatable, version=ver)
 
             if active_version:

glob/manager_server.py

@@ -566,7 +566,7 @@ async def fetch_customnode_list(request):
     else:
         channel = core.get_config()['channel_url']
 
-    node_packs = await core.get_unified_total_nodes(channel, request.rel_url.query["mode"])
+    node_packs = await core.get_unified_total_nodes(channel, request.rel_url.query["mode"], 'cache')
     json_obj_github = core.get_data_by_mode(request.rel_url.query["mode"], 'github-stats.json', 'default')
     json_obj_extras = core.get_data_by_mode(request.rel_url.query["mode"], 'extras.json', 'default')
 
@@ -1434,8 +1434,8 @@ async def get_cache(filename):
     await asyncio.gather(a, b, c, d, e)
 
     # load at least once
-    await core.unified_manager.reload('cache', dont_wait=False)
-    await core.unified_manager.get_custom_nodes('default', 'cache')
+    await core.unified_manager.reload('remote', dont_wait=False)
+    await core.unified_manager.get_custom_nodes('default', 'remote')
 
     # NOTE: hide migration button temporarily.
     # if not core.get_config()['skip_migration_check']:

glob/manager_util.py

@@ -130,9 +130,34 @@ async def get_data(uri, silent=False):
     return json_obj
 
 
-async def get_data_with_cache(uri, silent=False, cache_mode=True, dont_wait=False):
+def get_cache_path(uri):
     cache_uri = str(simple_hash(uri)) + '_' + os.path.basename(uri).replace('&', "_").replace('?', "_").replace('=', "_")
-    cache_uri = os.path.join(cache_dir, cache_uri+'.json')
+    return os.path.join(cache_dir, cache_uri+'.json')
+
+
+def get_cache_state(uri):
+    cache_uri = get_cache_path(uri)
+
+    if not os.path.exists(cache_uri):
+        return "not-cached"
+    elif is_file_created_within_one_day(cache_uri):
+        return "cached"
+
+    return "expired"
+
+
+def save_to_cache(uri, json_obj, silent=False):
+    cache_uri = get_cache_path(uri)
+
+    with cache_lock:
+        with open(cache_uri, "w", encoding='utf-8') as file:
+            json.dump(json_obj, file, indent=4, sort_keys=True)
+            if not silent:
+                logging.info(f"[ComfyUI-Manager] default cache updated: {uri}")
+
+
+async def get_data_with_cache(uri, silent=False, cache_mode=True, dont_wait=False):
+    cache_uri = get_cache_path(uri)
 
     if cache_mode and dont_wait:
         # NOTE: return the cache if possible, even if it is expired, so do not cache

js/custom-nodes-manager.js

@@ -975,7 +975,10 @@ export class CustomNodesManager {
 				}
 
 				const link = document.createElement('a');
-				link.href = rowItem.reference;
+				if(rowItem.originalData.repository)
+					link.href = rowItem.originalData.repository;
+				else
+					link.href = rowItem.reference;
 				link.target = '_blank';
 				link.innerHTML = `<b>${title}</b>`;
 				container.appendChild(link);

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-manager"
 description = "ComfyUI-Manager provides features to install and manage custom nodes for ComfyUI, as well as various functionalities to assist with ComfyUI."
-version = "3.8.1"
+version = "3.9"
 license = { file = "LICENSE.txt" }
 dependencies = ["GitPython", "PyGithub", "matrix-client==0.4.0", "transformers", "huggingface-hub>0.20", "typer", "rich", "typing-extensions"]