fix: scanner.py for python 3.12

update DB
This commit is contained in:
Dr.Lt.Data
2024-05-10 21:30:02 +09:00
parent 0f3e2418e0
commit 6a378052f2
6 changed files with 699 additions and 206 deletions

View File

@@ -37,12 +37,21 @@ else:
print(f"TEMP DIR: {temp_dir}")
parse_cnt = 0
def extract_nodes(code_text):
global parse_cnt
try:
if parse_cnt % 100 == 0:
print(f".", end="", flush=True)
parse_cnt += 1
code_text = re.sub(r'\\[^"\']', '', code_text)
parsed_code = ast.parse(code_text)
assignments = (node for node in parsed_code.body if isinstance(node, ast.Assign))
for assignment in assignments:
if isinstance(assignment.targets[0], ast.Name) and assignment.targets[0].id in ['NODE_CONFIG', 'NODE_CLASS_MAPPINGS']:
node_class_mappings = assignment.value
@@ -51,7 +60,12 @@ def extract_nodes(code_text):
node_class_mappings = None
if node_class_mappings:
s = set([key.s.strip() for key in node_class_mappings.keys if key is not None])
s = set()
for key in node_class_mappings.keys:
if key is not None and isinstance(key.value, str):
s.add(key.value.strip())
return s
else:
return set()
@@ -78,20 +92,24 @@ def scan_in_file(filename, is_builtin=False):
nodes |= extract_nodes(code)
pattern2 = r'^[^=]*_CLASS_MAPPINGS\["(.*?)"\]'
keys = re.findall(pattern2, code)
for key in keys:
nodes.add(key.strip())
def extract_keys(pattern, code):
keys = re.findall(pattern, code)
return {key.strip() for key in keys}
pattern3 = r'^[^=]*_CLASS_MAPPINGS\[\'(.*?)\'\]'
keys = re.findall(pattern3, code)
for key in keys:
nodes.add(key.strip())
def update_nodes(nodes, new_keys):
nodes |= new_keys
pattern4 = r'@register_node\("(.+)",\s*\".+"\)'
keys = re.findall(pattern4, code)
for key in keys:
nodes.add(key.strip())
patterns = [
r'^[^=]*_CLASS_MAPPINGS\["(.*?)"\]',
r'^[^=]*_CLASS_MAPPINGS\[\'(.*?)\'\]',
r'@register_node\("(.+)",\s*\".+"\)',
r'"(\w+)"\s*:\s*{"class":\s*\w+\s*'
]
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {executor.submit(extract_keys, pattern, code): pattern for pattern in patterns}
for future in concurrent.futures.as_completed(futures):
update_nodes(nodes, future.result())
matches = regex.findall(code)
for match in matches:
@@ -446,3 +464,4 @@ updated_node_info = update_custom_nodes()
print("\n# 'extension-node-map.json' file is generated.\n")
gen_json(updated_node_info)
print("\nDONE.\n")